{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" Train model for Study Level ","metadata":{}},{"cell_type":"code","source":"# Import the latest version of wandb\n# !pip install -q --upgrade wandb","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:28.628207Z","iopub.execute_input":"2021-08-09T06:46:28.628574Z","iopub.status.idle":"2021-08-09T06:46:28.633662Z","shell.execute_reply.started":"2021-08-09T06:46:28.628540Z","shell.execute_reply":"2021-08-09T06:46:28.632642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!/opt/conda/bin/python3.7 -m pip install --upgrade pip\n! pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:28.636354Z","iopub.execute_input":"2021-08-09T06:46:28.636875Z","iopub.status.idle":"2021-08-09T06:46:41.702114Z","shell.execute_reply.started":"2021-08-09T06:46:28.636837Z","shell.execute_reply":"2021-08-09T06:46:41.701026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:41.705661Z","iopub.execute_input":"2021-08-09T06:46:41.705947Z","iopub.status.idle":"2021-08-09T06:46:42.386690Z","shell.execute_reply.started":"2021-08-09T06:46:41.705916Z","shell.execute_reply":"2021-08-09T06:46:42.385769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import mixed_precision\n\n\n\nimport os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils.class_weight import compute_class_weight\n\n# Imports for augmentations. \nfrom albumentations import (Compose, RandomResizedCrop, Cutout, Rotate, HorizontalFlip, \n                            VerticalFlip, RandomBrightnessContrast, ShiftScaleRotate, \n                            CenterCrop, Resize)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.390155Z","iopub.execute_input":"2021-08-09T06:46:42.390446Z","iopub.status.idle":"2021-08-09T06:46:42.402115Z","shell.execute_reply.started":"2021-08-09T06:46:42.390414Z","shell.execute_reply":"2021-08-09T06:46:42.401205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GPU Run Access","metadata":{}},{"cell_type":"code","source":"# Increase GPU memory as per the need.\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-08-09T06:46:42.403843Z","iopub.execute_input":"2021-08-09T06:46:42.404421Z","iopub.status.idle":"2021-08-09T06:46:42.417016Z","shell.execute_reply.started":"2021-08-09T06:46:42.404384Z","shell.execute_reply":"2021-08-09T06:46:42.415968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_path='../input/siim-covid19-resized-to-256px-jpg/train/'  # for 256 img_size\n\n#Train_path='../input/siim-covid19-resized-to-512px-png/train/'   # for 512 img_size\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.418660Z","iopub.execute_input":"2021-08-09T06:46:42.419104Z","iopub.status.idle":"2021-08-09T06:46:42.425279Z","shell.execute_reply.started":"2021-08-09T06:46:42.419063Z","shell.execute_reply":"2021-08-09T06:46:42.424441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read df_train csv file\n\ndf=pd.read_csv('../input/df-train/df_train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.426788Z","iopub.execute_input":"2021-08-09T06:46:42.428849Z","iopub.status.idle":"2021-08-09T06:46:42.480388Z","shell.execute_reply.started":"2021-08-09T06:46:42.428817Z","shell.execute_reply":"2021-08-09T06:46:42.479592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read study_df from original dataset\n\ntrain_study_df=pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.481495Z","iopub.execute_input":"2021-08-09T06:46:42.481874Z","iopub.status.idle":"2021-08-09T06:46:42.494596Z","shell.execute_reply.started":"2021-08-09T06:46:42.481833Z","shell.execute_reply":"2021-08-09T06:46:42.493824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.498127Z","iopub.execute_input":"2021-08-09T06:46:42.498391Z","iopub.status.idle":"2021-08-09T06:46:42.512076Z","shell.execute_reply.started":"2021-08-09T06:46:42.498366Z","shell.execute_reply":"2021-08-09T06:46:42.511297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using study_id in train_df beacuse original files don't have the study_id and del all non-useable  columns \n# for train this model","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.514207Z","iopub.execute_input":"2021-08-09T06:46:42.514589Z","iopub.status.idle":"2021-08-09T06:46:42.518451Z","shell.execute_reply.started":"2021-08-09T06:46:42.514552Z","shell.execute_reply":"2021-08-09T06:46:42.517431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_df= train_study_df.drop(['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.520100Z","iopub.execute_input":"2021-08-09T06:46:42.520623Z","iopub.status.idle":"2021-08-09T06:46:42.528311Z","shell.execute_reply.started":"2021-08-09T06:46:42.520527Z","shell.execute_reply":"2021-08-09T06:46:42.527417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_df['StudyInstanceUID']=train_study_df['id'].apply(lambda x: x.replace('_study',''))","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.529703Z","iopub.execute_input":"2021-08-09T06:46:42.530074Z","iopub.status.idle":"2021-08-09T06:46:42.540282Z","shell.execute_reply.started":"2021-08-09T06:46:42.530019Z","shell.execute_reply":"2021-08-09T06:46:42.539564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_study_df=train_study_df.rename(columns={'id':'study_id'})\ntrain_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.541753Z","iopub.execute_input":"2021-08-09T06:46:42.542156Z","iopub.status.idle":"2021-08-09T06:46:42.556530Z","shell.execute_reply.started":"2021-08-09T06:46:42.542078Z","shell.execute_reply":"2021-08-09T06:46:42.555678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter","metadata":{}},{"cell_type":"code","source":"df=df.merge(train_study_df,on='StudyInstanceUID')\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.557745Z","iopub.execute_input":"2021-08-09T06:46:42.558308Z","iopub.status.idle":"2021-08-09T06:46:42.579454Z","shell.execute_reply.started":"2021-08-09T06:46:42.558267Z","shell.execute_reply":"2021-08-09T06:46:42.578705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.581173Z","iopub.execute_input":"2021-08-09T06:46:42.581777Z","iopub.status.idle":"2021-08-09T06:46:42.609582Z","shell.execute_reply.started":"2021-08-09T06:46:42.581738Z","shell.execute_reply":"2021-08-09T06:46:42.608575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['path']=df.apply(lambda row:Train_path + row.id +'.jpg',axis=1)  # for 256 img_size\n\n#df['path']=df.apply(lambda row:Train_path + row.id +'.png',axis=1)   # for 512 img_size","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.610963Z","iopub.execute_input":"2021-08-09T06:46:42.611345Z","iopub.status.idle":"2021-08-09T06:46:42.717088Z","shell.execute_reply.started":"2021-08-09T06:46:42.611307Z","shell.execute_reply":"2021-08-09T06:46:42.716304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['path'][0]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.718241Z","iopub.execute_input":"2021-08-09T06:46:42.718612Z","iopub.status.idle":"2021-08-09T06:46:42.724536Z","shell.execute_reply.started":"2021-08-09T06:46:42.718574Z","shell.execute_reply":"2021-08-09T06:46:42.723645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('000a312787f2.jpg' in os.listdir(Train_path))","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.726094Z","iopub.execute_input":"2021-08-09T06:46:42.726690Z","iopub.status.idle":"2021-08-09T06:46:42.737338Z","shell.execute_reply.started":"2021-08-09T06:46:42.726651Z","shell.execute_reply":"2021-08-09T06:46:42.736078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = df[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']].values\nlabels = np.argmax(labels, axis=1)\ndf['study_level'] = labels","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.738722Z","iopub.execute_input":"2021-08-09T06:46:42.739282Z","iopub.status.idle":"2021-08-09T06:46:42.747459Z","shell.execute_reply.started":"2021-08-09T06:46:42.739239Z","shell.execute_reply":"2021-08-09T06:46:42.746529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.drop(['StudyInstanceUID_count'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.748803Z","iopub.execute_input":"2021-08-09T06:46:42.749388Z","iopub.status.idle":"2021-08-09T06:46:42.758107Z","shell.execute_reply.started":"2021-08-09T06:46:42.749352Z","shell.execute_reply":"2021-08-09T06:46:42.757096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.759598Z","iopub.execute_input":"2021-08-09T06:46:42.760032Z","iopub.status.idle":"2021-08-09T06:46:42.768795Z","shell.execute_reply.started":"2021-08-09T06:46:42.759989Z","shell.execute_reply":"2021-08-09T06:46:42.767741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/siim-covid19-resized-to-256px-jpg/train/'\n\nTEST_PATH = '../input/siim-covid19-resized-to-256px-jpg/test/'\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.770211Z","iopub.execute_input":"2021-08-09T06:46:42.770750Z","iopub.status.idle":"2021-08-09T06:46:42.776477Z","shell.execute_reply.started":"2021-08-09T06:46:42.770707Z","shell.execute_reply":"2021-08-09T06:46:42.775318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for 512 images\n\n#TRAIN_PATH='../input/siim-covid19-resized-to-512px-png/train/'\n#TEST_PATH ='../input/siim-covid19-resized-to-512px-png/test/'","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.778286Z","iopub.execute_input":"2021-08-09T06:46:42.778710Z","iopub.status.idle":"2021-08-09T06:46:42.783808Z","shell.execute_reply.started":"2021-08-09T06:46:42.778612Z","shell.execute_reply":"2021-08-09T06:46:42.782855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.788115Z","iopub.execute_input":"2021-08-09T06:46:42.788420Z","iopub.status.idle":"2021-08-09T06:46:42.793582Z","shell.execute_reply.started":"2021-08-09T06:46:42.788395Z","shell.execute_reply":"2021-08-09T06:46:42.792289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df=train_df[['study_id',\n                  'Negative for Pneumonia','Typical Appearance',\n                  'Indeterminate Appearance','Atypical Appearance']]\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.795921Z","iopub.execute_input":"2021-08-09T06:46:42.796296Z","iopub.status.idle":"2021-08-09T06:46:42.802467Z","shell.execute_reply.started":"2021-08-09T06:46:42.796261Z","shell.execute_reply":"2021-08-09T06:46:42.801703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.803918Z","iopub.execute_input":"2021-08-09T06:46:42.804351Z","iopub.status.idle":"2021-08-09T06:46:42.816870Z","shell.execute_reply.started":"2021-08-09T06:46:42.804316Z","shell.execute_reply":"2021-08-09T06:46:42.815929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df= train_df.drop(['Unnamed: 0','boxes','label','index','0','Path', 'w', 'h', 'class','x_max', 'x_min', 'y_min',\n               'label_int','OpacityCount', 'y_max','study_id','Negative for Pneumonia',\n               'Typical Appearance','Indeterminate Appearance','Atypical Appearance'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.818178Z","iopub.execute_input":"2021-08-09T06:46:42.818538Z","iopub.status.idle":"2021-08-09T06:46:42.825074Z","shell.execute_reply.started":"2021-08-09T06:46:42.818482Z","shell.execute_reply":"2021-08-09T06:46:42.823852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.826467Z","iopub.execute_input":"2021-08-09T06:46:42.826841Z","iopub.status.idle":"2021-08-09T06:46:42.839704Z","shell.execute_reply.started":"2021-08-09T06:46:42.826805Z","shell.execute_reply":"2021-08-09T06:46:42.838747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.840923Z","iopub.execute_input":"2021-08-09T06:46:42.841320Z","iopub.status.idle":"2021-08-09T06:46:42.848327Z","shell.execute_reply.started":"2021-08-09T06:46:42.841284Z","shell.execute_reply":"2021-08-09T06:46:42.847554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 4\nHEIGHT,WIDTH = 256,256\n#HEIGHT,WIDTH = 512,512\n\nCHANNELS = 3\nBATCH_SIZE = 8\nSEED = 143","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.849420Z","iopub.execute_input":"2021-08-09T06:46:42.849739Z","iopub.status.idle":"2021-08-09T06:46:42.857814Z","shell.execute_reply.started":"2021-08-09T06:46:42.849707Z","shell.execute_reply":"2021-08-09T06:46:42.856853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_img(filepath,label):\n    image = tf.io.read_file(filepath)\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = tf.image.convert_image_dtype(image, tf.float32) \n    #image = tf.image.resize(image, [HEIGHT,WIDTH])\n    return image,label\n\n\ndef data_augment(image, label):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) \n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) \n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) \n        \n    \n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    \n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n    \n    image = tf.image.resize(image, [HEIGHT,WIDTH])\n    return image,label\n\ndef get_dataset(filenames,labels, training=True):\n    dataset = tf.data.Dataset.from_tensor_slices((filenames,labels))\n    dataset = dataset.map(process_img,num_parallel_calls=AUTO)\n    dataset = dataset.map(data_augment,num_parallel_calls=AUTO)\n    dataset = dataset.cache()\n    dataset = dataset.repeat()\n    if training:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.859278Z","iopub.execute_input":"2021-08-09T06:46:42.859782Z","iopub.status.idle":"2021-08-09T06:46:42.877419Z","shell.execute_reply.started":"2021-08-09T06:46:42.859736Z","shell.execute_reply":"2021-08-09T06:46:42.876588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\n#def create_model():\n    \n#    pretrained = efn.EfficientNetB4(include_top=False, weights=\"imagenet\",input_shape=[HEIGHT,WIDTH, 3])\n            \n#    x = pretrained.output\n#    x = tf.keras.layers.GlobalAveragePooling2D() (x)\n#    outputs = tf.keras.layers.Dense(NUM_CLASSES,activation=\"softmax\", dtype='float32')(x)\n        \n#    model = tf.keras.Model(pretrained.input, outputs)\n#    return model\n\n#model = create_model()\n#model.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.878668Z","iopub.execute_input":"2021-08-09T06:46:42.879215Z","iopub.status.idle":"2021-08-09T06:46:42.888059Z","shell.execute_reply.started":"2021-08-09T06:46:42.879156Z","shell.execute_reply":"2021-08-09T06:46:42.887209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    \n    base_model = efn.EfficientNetB0(include_top=False, weights='imagenet')\n    base_model.trainabe = True\n\n    inputs = layers.Input((HEIGHT,WIDTH, 3))\n    x = base_model(inputs, training=True)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.5)(x)\n    \n    outputs = layers.Dense(NUM_CLASSES, kernel_regularizer=tf.keras.regularizers.l2(0.001))(x)\n    outputs = layers.Activation('softmax', dtype='float32', name='predictions')(outputs)\n    \n    return models.Model(inputs, outputs)\n\ntf.keras.backend.clear_session()\n#model = create_model()\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.890824Z","iopub.execute_input":"2021-08-09T06:46:42.891075Z","iopub.status.idle":"2021-08-09T06:46:42.902326Z","shell.execute_reply.started":"2021-08-09T06:46:42.891043Z","shell.execute_reply":"2021-08-09T06:46:42.901378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_addons as tfa\n\ndef compile_model(model, lr=0.001):\n    \n    optimizer = tf.keras.optimizers.Adam(lr=lr)\n    \n    loss = tf.keras.losses.CategoricalCrossentropy()\n   \n    metrics = [\n       tfa.metrics.F1Score(num_classes = NUM_CLASSES,average = \"macro\", name = \"f1_score\"),\n       tf.keras.metrics.CategoricalAccuracy(name='acc')\n    ]\n\n    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.904449Z","iopub.execute_input":"2021-08-09T06:46:42.905196Z","iopub.status.idle":"2021-08-09T06:46:42.911250Z","shell.execute_reply.started":"2021-08-09T06:46:42.905159Z","shell.execute_reply":"2021-08-09T06:46:42.910394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"METRIC = \"val_acc\"\n# METRIC=\"val_auc\"\ndef create_callbacks(kfold,metric = METRIC):\n    \n    cpk_path = f'./best_model_{kfold}.h5'\n    \n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        filepath=cpk_path,\n        monitor= metric,\n        mode='max',\n        save_best_only=True,\n        verbose=1,\n    )\n\n    reducelr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor= metric,\n        mode='max',\n        factor=0.1,\n        patience=3,\n        verbose=0\n    )\n\n    earlystop = tf.keras.callbacks.EarlyStopping(\n        monitor= metric,\n        mode='max',\n        patience=10, \n        verbose=1\n    )\n    \n    callbacks = [checkpoint, reducelr, earlystop]         \n    \n    return callbacks","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.912665Z","iopub.execute_input":"2021-08-09T06:46:42.913100Z","iopub.status.idle":"2021-08-09T06:46:42.922063Z","shell.execute_reply.started":"2021-08-09T06:46:42.912992Z","shell.execute_reply":"2021-08-09T06:46:42.921220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)\ntrain_df.to_csv('train_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.923818Z","iopub.execute_input":"2021-08-09T06:46:42.924308Z","iopub.status.idle":"2021-08-09T06:46:42.972689Z","shell.execute_reply.started":"2021-08-09T06:46:42.924268Z","shell.execute_reply":"2021-08-09T06:46:42.971937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head(3)\nstudy_df.to_csv('study_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:42.973855Z","iopub.execute_input":"2021-08-09T06:46:42.974200Z","iopub.status.idle":"2021-08-09T06:46:42.999663Z","shell.execute_reply.started":"2021-08-09T06:46:42.974164Z","shell.execute_reply":"2021-08-09T06:46:42.998916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nfiles_ls= train_df['path']\n\nfiles_df = pd.DataFrame(list(files_ls), columns = [\"filepath\"])\n\n\nlabels = np.zeros((len(files_ls),NUM_CLASSES))\ntmp_labels = np.zeros((len(files_ls)))\n\ndef get_id(filepath):\n    tmp = filepath.split(\"/\")[-1]\n    tmp = tmp.split(\".\")[0]\n    tmp = tmp.split(\"_\")[-1]\n    return tmp\n\nfor i in tqdm(range(len(files_ls))):\n    image_id = get_id(files_ls[i])\n    label_id = train_df[train_df[\"id\"] == image_id][\"study_level\"]\n    labels[i][label_id] = 1\n    tmp_labels[i] = label_id\n    \nprint(\"Labels shape: \",labels.shape)\nprint(files_ls)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:43.000788Z","iopub.execute_input":"2021-08-09T06:46:43.001185Z","iopub.status.idle":"2021-08-09T06:46:52.940276Z","shell.execute_reply.started":"2021-08-09T06:46:43.001145Z","shell.execute_reply":"2021-08-09T06:46:52.939279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 30\nVERBOSE = 1\nN_SPLITS = 5\n\nkfold = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\nhistory = {}\n\n\nfor fold,(tID,vID) in enumerate(kfold.split(files_ls,tmp_labels)):\n    tFiles, tLabels = list(files_df.iloc[tID][\"filepath\"]) , labels[tID]\n    vFiles, vLabels = list(files_df.iloc[vID][\"filepath\"]) , labels[vID]\n    print(\"Number of Training Images: \",len(tID))\n    print(\"Number of Validation Images: \",len(vID))\n    \n    STEPS_PER_EPOCH  = len(tID)//BATCH_SIZE\n    VALID_STEPS = len(vID)//BATCH_SIZE\n    \n    tf.keras.backend.clear_session()\n    \n    train_ds = get_dataset(tFiles,tLabels, training = True)\n    val_ds = get_dataset(vFiles, vLabels, training = False)\n    \n    \n\n    model = create_model()\n    #model = chxnet\n    model = compile_model(model, lr=0.00001)\n    callbacks = create_callbacks(kfold = fold)\n\n    print(\"------------------Fold - \",fold+1,\" --------------------------\")\n    history[fold] = model.fit(\n                        train_ds,\n                        epochs=EPOCHS,\n                        callbacks=callbacks,\n                        validation_data = val_ds,\n                        verbose=VERBOSE,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        validation_steps=VALID_STEPS\n                       )","metadata":{"execution":{"iopub.status.busy":"2021-08-09T06:46:52.941622Z","iopub.execute_input":"2021-08-09T06:46:52.942122Z","iopub.status.idle":"2021-08-09T08:48:35.297976Z","shell.execute_reply.started":"2021-08-09T06:46:52.942078Z","shell.execute_reply":"2021-08-09T08:48:35.296925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8*N_SPLITS,24))\n\nfor i in range(N_SPLITS):\n    acc = history[i].history['acc']\n    val_acc = history[i].history['val_acc']\n    f1 = history[i].history['f1_score']\n    val_f1 = history[i].history['val_f1_score']\n    loss = history[i].history['loss']\n    val_loss = history[i].history['val_loss']\n    epochs_range = range(len(history[i].history['val_loss'])) \n    \n    plt.subplot(N_SPLITS, 3,i*3+1)\n    plt.plot(epochs_range, acc, label='Training Accuracy')\n    plt.plot(epochs_range, val_acc, label='Validation  Accuracy')\n    plt.legend(loc='lower right')\n    plt.title(f'FOLD:{str(i)} Training and Validation  Accuracy')\n    \n    plt.subplot(N_SPLITS, 3,i*3+2)\n    plt.plot(epochs_range, f1, label='Training F1 score')\n    plt.plot(epochs_range, val_f1, label='Validation  F1 score')\n    plt.legend(loc='lower right')\n    plt.title(f'FOLD:{str(i)} Training and Validation  F1 score')\n    \n    plt.subplot(N_SPLITS, 3, i*3+3)\n    plt.plot(epochs_range, loss, label='Training Loss')\n    plt.plot(epochs_range, val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.title(f'FOLD:{str(i)} Training and Validation Loss')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T08:48:35.304299Z","iopub.execute_input":"2021-08-09T08:48:35.304670Z","iopub.status.idle":"2021-08-09T08:48:37.492816Z","shell.execute_reply.started":"2021-08-09T08:48:35.304631Z","shell.execute_reply":"2021-08-09T08:48:37.490018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nBATCH_SIZE = 16","metadata":{"execution":{"iopub.status.busy":"2021-08-09T08:48:37.494140Z","iopub.execute_input":"2021-08-09T08:48:37.494443Z","iopub.status.idle":"2021-08-09T08:48:37.499372Z","shell.execute_reply.started":"2021-08-09T08:48:37.494412Z","shell.execute_reply":"2021-08-09T08:48:37.498109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"EPOCHS = 30\nVERBOSE = 1\nN_SPLITS = 5\n\nkfold = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\nhistory = {}\n\n\n\n        \n\nfor fold,(tID,vID) in enumerate(kfold.split(files_ls,tmp_labels)):\n    tFiles, tLabels = list(files_df.iloc[tID][\"filepath\"]) , labels[tID]\n    vFiles, vLabels = list(files_df.iloc[vID][\"filepath\"]) , labels[vID]\n    print(\"Number of Training Images: \",len(tID))\n    print(\"Number of Validation Images: \",len(vID))\n    \n    STEPS_PER_EPOCH  = len(tID)//BATCH_SIZE\n    VALID_STEPS = len(vID)//BATCH_SIZE\n    \n    tf.keras.backend.clear_session()\n    \n    train_ds = get_dataset(tFiles,tLabels, training = True)\n    val_ds = get_dataset(vFiles, vLabels, training = False)\n    \n        \n    model = tf.keras.Sequential([\n        efn.EfficientNetB5(\n            input_shape=(HEIGHT, WIDTH, 3),\n            weights='imagenet',\n            include_top=False),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n    ])\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss='categorical_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n    callbacks = create_callbacks(kfold = fold)\n    print(\"------------------Fold - \",fold+1,\" --------------------------\")\n    history[fold] = model.fit(\n                        train_ds,\n                        epochs=EPOCHS,\n                        callbacks=callbacks,\n                        validation_data = val_ds,\n                        verbose=VERBOSE,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        validation_steps=VALID_STEPS\n                       )","metadata":{"execution":{"iopub.status.busy":"2021-08-09T02:22:00.076437Z","iopub.execute_input":"2021-08-09T02:22:00.076809Z"}}}]}