{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport sys\nimport copy\nimport cv2\n\nimport types\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom skimage import exposure\n\nimport tensorflow as tf \nfrom tensorflow import keras\nfrom tensorflow.keras.applications import ResNet50, DenseNet121, Xception,InceptionResNetV2,VGG16\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout, GlobalAveragePooling2D, Input\nfrom tensorflow.keras import optimizers,losses,metrics\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping, Callback\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model\nimport seaborn as sns\n\nimport math\nimport matplotlib.pyplot as plt\n\n\ninputdir = \"../input/siim-covid19-detection\"\nimagedir = \"../input/covid-jpg-512\"\n\n# Helper Function to shuffle the np rows independently. \ndef shuffle_rows_independently(data_np):\n    rows = data_np.shape[0]\n    for i in range(rows):\n        np.random.shuffle(data_np[i])\n    return data_np\n\n# Function to shuffle the opacity values to avoid overfitting during training\ndef shuffle_opacity_locations(df):\n    max_boxes = int(df['ImageBoxCount'].max())\n    drop_columns = list()\n    for i in range(max_boxes):\n        df['X_old'+str(i+1)] = df['X'+str(i+1)]\n        df['Y_old'+str(i+1)] = df['Y'+str(i+1)]\n        df['W_old'+str(i+1)] = df['W'+str(i+1)]\n        df['H_old'+str(i+1)] = df['H'+str(i+1)]\n        df['I'+str(i+1)] = i+1\n\n        drop_columns.append('X_old'+str(i+1))\n        drop_columns.append('Y_old'+str(i+1))\n        drop_columns.append('W_old'+str(i+1))\n        drop_columns.append('H_old'+str(i+1))\n        drop_columns.append('I'+str(i+1))\n\n    for i in range(max_boxes):\n        rows = df[df['ImageBoxCount'] == i+1].shape[0]\n        cols = i+1\n        shuffle_np = np.zeros(shape=(rows,cols),dtype=int)\n        shuffle_cols = list()\n        for j in range(i+1):\n            shuffle_cols.append('I'+str(j+1))\n        shuffle_np = np.tile(np.arange(1,i+2),(rows,1))\n        shuffle_np = shuffle_rows_independently(shuffle_np)\n        #print(\"roots = %d, columns = %s shape = %s, \\nshuffle_np = \\n%s\" %(i+1,shuffle_cols,shuffle_np.shape,shuffle_np))\n        for j in range(i+1):\n            df.loc[df['ImageBoxCount'] == i+1,'I'+str(j+1)] = shuffle_np[:,j]\n    for i in range(max_boxes):\n        for j in range(max_boxes):\n            df.loc[df['I'+str(j+1)] == i+1,'X'+str(i+1)] = df.loc[df['I'+str(j+1)] == i+1,'X_old'+str(j+1)]\n            df.loc[df['I'+str(j+1)] == i+1,'Y'+str(i+1)] = df.loc[df['I'+str(j+1)] == i+1,'Y_old'+str(j+1)]\n            df.loc[df['I'+str(j+1)] == i+1,'W'+str(i+1)] = df.loc[df['I'+str(j+1)] == i+1,'W_old'+str(j+1)]\n            df.loc[df['I'+str(j+1)] == i+1,'H'+str(i+1)] = df.loc[df['I'+str(j+1)] == i+1,'H_old'+str(j+1)]\n\n    return df.drop(columns=drop_columns)\n\n# Function to read Image and Study level CSV files, merge and preprocessing them in desired format\ndef read_dataset_csv(study_csv_path,image_csv_path,size_csv_path,study_columns=None,padding_value_xy=0.0,padding_value_wh=0.0,image_format='jpg'):\n    df_image = pd.read_csv(image_csv_path)\n    df_study = pd.read_csv(study_csv_path)\n    df_size = pd.read_csv(size_csv_path)\n\n    prefix = image_csv_path.split('/')[-1].split('_')[0]\n    prefix_study = study_csv_path.split('/')[-1].split('_')[0]\n    if prefix != prefix_study:\n        print(\"Info: Image level prefix and Study Level prefix don't match. Going with Prefix(%s) from Path(%s) and ignoring the Path(%s)\" %(prefix,image_csv_path,study_csv_path))\n\n    \n    \n    if study_columns is not None:\n        df_study.columns = study_columns\n        \n    df_study['StudyInstanceUID'] = df_study['id'].replace(to_replace='_study',value='',regex=True)\n    \n    df_merged = df_image.merge(df_study[df_study.columns.tolist()[1:]], on=['StudyInstanceUID'])\n    \n    df_merged['ImageInstanceUID'] = df_merged['id'].replace(to_replace='_image',value='',regex=True)\n    df_merged['ImageBoxCount'] = df_merged['label'].str.split(' ').str.len().divide(6)\n    df_merged.loc[df_merged['boxes'].isna(),'ImageBoxCount'] = 0.0\n    \n    df_merged['fname'] = df_merged['id'].str.replace('_image', '.' + image_format)\n    df_size = df_size.rename(columns={\"id\":\"fname\"})\n\n    df_merged = df_merged.merge(df_size[['fname','dim0','dim1']], on=['fname'])\n    max_boxes = int(df_merged['ImageBoxCount'].max())\n    \n    for i in range(max_boxes):\n        df_merged['X'+str(i+1)] = df_merged['label'].str.split(' ').str[(6*i)+2].fillna(0.0).astype(float)\n        df_merged['Y'+str(i+1)] = df_merged['label'].str.split(' ').str[(6*i)+3].fillna(0.0).astype(float)\n        df_merged['W'+str(i+1)] = df_merged['label'].str.split(' ').str[(6*i)+4].fillna(0.0).astype(float) - df_merged['X'+str(i+1)]\n        df_merged['H'+str(i+1)] = df_merged['label'].str.split(' ').str[(6*i)+5].fillna(0.0).astype(float) - df_merged['Y'+str(i+1)]\n        \n        df_merged.loc[df_merged['ImageBoxCount'] < i+1, 'X'+str(i+1)] = padding_value_xy\n        df_merged.loc[df_merged['ImageBoxCount'] < i+1, 'Y'+str(i+1)] = padding_value_xy\n        df_merged.loc[df_merged['ImageBoxCount'] < i+1, 'W'+str(i+1)] = padding_value_wh\n        df_merged.loc[df_merged['ImageBoxCount'] < i+1, 'H'+str(i+1)] = padding_value_wh\n\n        df_merged['X'+str(i+1)] = df_merged['X'+str(i+1)] / df_merged['dim0'] \n        df_merged['Y'+str(i+1)] = df_merged['X'+str(i+1)] / df_merged['dim1']\n        df_merged['W'+str(i+1)] = df_merged['X'+str(i+1)] / df_merged['dim0']\n        df_merged['H'+str(i+1)] = df_merged['X'+str(i+1)] / df_merged['dim1']\n        \n    df_merged = df_merged.fillna(0.0)\n    study_labels = ['Negative','Typical','Indeterminate','Atypical']\n    df_merged['study_label'] = 'UnAssigned'\n    i = 0\n    for column in df_study.columns.tolist()[1:-1]:\n        df_merged.loc[df_merged[column]==1, 'study_label'] = study_labels[i]\n        i += 1\n    \n\n    df_merged = df_merged.rename(columns={\"label\":\"image_label\"})\n    return df_merged\n\n\n# Function to read image in dicom format and convert to a array (Code resuse from Sina)\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\n# Function for image pre-processing (Code resuse from Sina)\ndef preprocess_image(img):\n    equ_img = exposure.equalize_hist(img)\n    return equ_img\n\n\n# Function for reading image from the directory\ndef get_image_generator(inputdf,dirpath,x_col,y_col,img_x,img_y,batch_size,validation_split,subset):\n\n    if type(y_col) is not list or len(y_col) == 1:\n        class_mode='categorical'\n    else:\n        class_mode = 'multi_output'\n    if type(y_col) is list and len(y_col) == 1:\n        y_col = y_col[0]\n\n    if subset != 'validation':\n        image_generator = ImageDataGenerator(\n            validation_split=validation_split,\n            #rotation_range=20,\n            horizontal_flip = True,\n            zoom_range = 0.1,\n            #shear_range = 0.1,\n            brightness_range = [0.8, 1.1],\n            fill_mode='nearest',\n            preprocessing_function=preprocess_image,\n        )\n    else:\n        image_generator = ImageDataGenerator(\n            validation_split=validation_split,\n            preprocessing_function=preprocess_image,\n        )\n\n    dataset = image_generator.flow_from_dataframe(\n        dataframe = inputdf,\n        directory=dirpath,\n        x_col = x_col,\n        y_col =  y_col,  \n        target_size=(img_x,img_y),\n        batch_size=batch_size,\n        subset= subset,\n        class_mode=class_mode\n    ) \n    \n    return dataset\n\ndef iterator_to_numpy(inputdf,dirpath,x_col,y_col,img_x,img_y,batch_size,validation_split,subset,datagen=None):\n    if datagen is None:\n        datagen = get_image_generator(inputdf,dirpath,x_col,y_col,img_x,img_y,batch_size,validation_split,subset)\n    samples = len(datagen.filenames)\n    dataX = None\n    dataY = None\n\n    i = 0\n    while i < samples:\n        batchX,batchY = datagen.next()\n        if datagen.class_mode != 'categorical':\n            batchY = np.transpose(np.array(batchY))\n        if dataX is None:\n            dataX = batchX\n            dataY = batchY\n        else: \n            dataX = np.concatenate((dataX,batchX),axis=0)\n            dataY = np.concatenate((dataY,batchY),axis=0)\n        i += batchX.shape[0]\n    return (dataX,dataY)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-11T18:45:53.886011Z","iopub.execute_input":"2021-06-11T18:45:53.886432Z","iopub.status.idle":"2021-06-11T18:45:53.937223Z","shell.execute_reply.started":"2021-06-11T18:45:53.886389Z","shell.execute_reply":"2021-06-11T18:45:53.936179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\ndef writeFiltersSummary(model,outpath,title):\n    outfile = open(outpath, 'w')\n    outfile.write(\"%s: 2D-Convolution  Layer Summary\" %(title))\n    for layer in model.layers:\n        if 'conv2d' in layer.name:\n            filters = np.array(layer.get_weights())[0]\n            outfile.write(\"layerName = %s, filtersShape = %s\\n\" %(layer.name,filters.shape))\n    outfile.close()\n\ndef visualizeFilters(model,title,layerName,rows = 3, cols = 6):\n    layer = model.get_layer(name=layerName)\n    filters = np.array(layer.get_weights())[0]\n    f_min, f_max = filters.min(), filters.max()\n    filters = (filters - f_min) / (f_max - f_min)\n    index = 1\n    fig = plt.figure()\n    fig.suptitle(\"%s 2D-Convolution Layer Filters: %s\" %(title,layerName), fontsize=16)\n    for r in range(rows):\n        for c in range(cols):\n            ax = plt.subplot(rows,cols,index)\n            ax.set_xticks([])\n            ax.set_yticks([])\n            ax.set_title(\"filter(\" + str(r) + \",\" + str(c) + \")\")\n            plt.imshow(filters[:,:,r,c], cmap='gray')\n            index += 1\n\n    plt.show()\n\ndef writeModelSummary(model,outpath,title):\n    outfile = open(outpath, 'w')\n    outfile.write(\"%s Model Summary\" %(title))\n    model.summary(print_fn=lambda x: outfile.write(x + '\\n'))\n    outfile.close()\n\ndef show_model_metrics(model,inputs_test, targets_test,inputs_train, targets_train,title=None,classes=['Negative','Typical','Indeterminate','Atypical'],batch_size=256,threshold_wh=1e-6,generator_train=None,generator_test=None):\n    loss_function = tf.keras.losses.MeanSquaredError()  \n    if inputs_test is not None:\n        predicted = model.predict(inputs_test)\n        actual = targets_test\n        #loss_test_eval,accuracy_test_eval = model.evaluate(inputs_test,targets_test,batch_size=batch_size,verbose=0)\n    else:\n        predicted = model.predict(generator_test)\n        actual = np.transpose(np.array(generator_train.labels))\n        #loss_test_eval,accuracy_test_eval = model.evaluate(self.generator_test,batch_size=batch_size,verbose=0)\n\n    predicted_study = np.argmax(predicted[:,:4],axis=1)\n    actual_study = np.argmax(actual[:,:4],axis=1)\n\n    hits_study = len(np.where((actual_study - predicted_study) == 0)[0])\n    accuracy_test_study = (hits_study/actual.shape[0])\n    #loss_test_study = loss_function(actual_study,predicted_study).numpy()\n\n    confusion_matrix = tf.math.confusion_matrix(labels=actual_study,predictions=predicted_study)\n    confusion_matrix_numpy = np.array(confusion_matrix)\n    confusion_matrix_normalized = np.around(confusion_matrix_numpy.astype('float') / confusion_matrix_numpy.sum(axis=1)[:,np.newaxis],decimals=3)\n    confusion_matrix_test = pd.DataFrame(confusion_matrix_normalized,index=classes,columns=classes)\n\n    if inputs_train is not None:\n        predicted = model.predict(inputs_train)\n        actual = targets_train\n        #loss_train_eval,accuracy_train_eval = model.evaluate(inputs_train,targets_train,batch_size=batch_size,verbose=0)\n    else:\n        predicted = model.predict(generator_train)\n        actual = np.transpose(np.array(generator_train.labels))\n        #loss_train_eval,accuracy_train_eval = model.evaluate(self.generator_train,batch_size=batch_size,verbose=0)\n\n    predicted_study = np.argmax(predicted[:,:4],axis=1)\n    actual_study = np.argmax(actual[:,:4],axis=1)\n\n    hits_study = len(np.where((actual_study - predicted_study) == 0)[0])\n    accuracy_train_study = (hits_study/actual.shape[0])\n    #loss_train_study = loss_function(actual_study,predicted_study).numpy()\n\n    fig =  plt.figure(figsize=(6, 6))\n    sns.heatmap(confusion_matrix_test, annot=True,cmap=plt.cm.Blues)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    if title and title != \"\":\n        #fig.suptitle(\"%s \\n Model Accuracy Percentage: Test = %.2f Training = %.2f \\n\\n Model Confusion Matrix\" % (title,accuracy_test_study*100.0,accuracy_train_study*100.0), fontsize=16)\n        fig.suptitle(\"Confusion Matrix\")\n    else:\n        print(\"%s: Model Accuracy Percentage: Test = %.2f Training = %.2f \\n\\n Model Confusion Matrix\" % (title,accuracy_test_study*100.0,accuracy_train_study*100.0))\n        fig.suptitle(\"Model Confusion Matrix\", fontsize=16)\n\n    plt.show()\n\ndef plot_per_epoch_data(data,title,tag=[\"study\",\"eval\"], plot_loss=False):\n    x1 = data[\"test-accuracy:\" + tag[0]].keys()\n    y1 = data[\"test-accuracy:\" + tag[0]].values()\n\n    x2 = data[\"train-accuracy:\" + tag[0]].keys()\n    y2 = data[\"train-accuracy:\" + tag[0]].values()\n\n    plt.plot(x1, y1, 'b', label='Test Accuracy')\n    plt.plot(x2, y2, 'r', label='Training Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy(%)')\n    plt.title(title + \" :Accuracy\")\n    plt.legend(loc='lower right')\n    plt.show()\n\n    if plot_loss:\n        x1 = data[\"test-loss:\" + tag[1]].keys()\n        y1 = data[\"test-loss:\" + tag[1]].values()\n\n        x2 = data[\"train-loss:\" + tag[1]].keys()\n        y2 = data[\"train-loss:\" + tag[1]].values()\n\n        plt.plot(x1, y1, 'b', label='Test Loss')\n        plt.plot(x2, y2, 'r', label='Training Loss')\n        plt.xlabel('Epoch')\n        plt.ylabel('loss')\n        plt.title(title + \" :Loss\")\n        plt.legend(loc='upper center')\n        plt.show()\n\n\nclass TestCallback(Callback):\n    def __init__(self,inputs_test, targets_test,inputs_train, targets_train,outdata,validation_freq=10,batch_size=256,threshold_wh=1e-6,generator_train=None,generator_test=None):\n        self.inputs_test = inputs_test\n        self.targets_test = targets_test\n        self.inputs_train = inputs_train\n        self.targets_train = targets_train\n        self.threshold_wh = threshold_wh\n        self.outdata = outdata\n        self.validation_freq = validation_freq\n        self.batch_size = batch_size\n        self.generator_train = generator_train\n        self.generator_test = generator_test\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch < self.validation_freq or epoch % self.validation_freq == 0:\n            loss_function = tf.keras.losses.MeanSquaredError()  \n            if self.inputs_train is not None:\n                predicted = self.model.predict(self.inputs_train)\n                actual = np.array(self.targets_train)\n                #loss_train_eval,accuracy_train_eval = self.model.evaluate(self.inputs_train,self.targets_train,verbose=0)\n\n            else:\n                predicted = self.model.predict(self.generator_train)\n                actual = np.transpose(np.array(self.generator_train))\n                #loss_train_eval,accuracy_train_eval = self.model.evaluate(self.generator_train,verbose=0)\n\n            predicted_study = np.argmax(predicted[:,:4],axis=1)\n            actual_study = np.argmax(actual[:,:4],axis=1)\n            hits_study = len(np.where((actual_study - predicted_study) == 0)[0])\n            accuracy_train_study = (hits_study/actual.shape[0])\n            #loss_train_study = loss_function(actual_study,predicted_study).numpy()\n\n            if self.inputs_test is not None:\n                predicted = self.model.predict(self.inputs_test)\n                actual = self.targets_test\n                #loss_test_eval,accuracy_test_eval = self.model.evaluate(self.inputs_test,self.targets_test,batch_size=batch_size,verbose=0)\n\n            else:\n                predicted = self.model.predict(self.generator_test)\n                actual = np.transpose(np.array(self.generator_test))\n                #loss_test_eval,accuracy_test_eval = self.model.evaluate(self.generator_test,batch_size=batch_size,verbose=0)\n\n            predicted_study = np.argmax(predicted[:,:4],axis=1)\n            actual_study = np.argmax(actual[:,:4],axis=1)\n            hits_study = len(np.where((predicted_study - actual_study) == 0)[0])\n            accuracy_test_study = (hits_study/actual.shape[0])\n            #loss_test_study = loss_function(actual_study,predicted_study).numpy()\n\n            print(\"================================== Epoch(%d) Custom Metrics: Start ==================================\" % (epoch+1))\n            print(\"\\t\\tTest Accuracy(study) = %.3f, Training Accuracy(study) = %.3f\" % (accuracy_test_study*100.0,accuracy_train_study*100.0))\n            #print(\"\\t\\tTest Loss(study) = %.3f, Training Loss(study) = %.3f\" % (loss_test_study,loss_train_study))\n            #print(\"\\t\\tTest Accuracy(eval) = %.3f, Training Accuracy(eval) = %.3f\" % (accuracy_test_eval*100.0,accuracy_train_eval*100.0))\n            #print(\"\\t\\tTest Loss(eval) = %.3f, Training Loss(eval) = %.3f\" % (loss_test_eval,loss_train_eval))\n\n            self.outdata[\"test-accuracy:study\"][epoch+1] = accuracy_test_study * 100.0\n            self.outdata[\"train-accuracy:study\"][epoch+1] = accuracy_train_study * 100.0\n            #self.outdata[\"test-loss:study\"][epoch+1] = loss_test_study\n            #self.outdata[\"train-loss:study\"][epoch+1] = loss_train_study\n\n            #self.outdata[\"test-accuracy:eval\"][epoch+1] = accuracy_test_eval * 100.0\n            #self.outdata[\"train-accuracy:eval\"][epoch+1] = accuracy_train_eval * 100.0\n            #self.outdata[\"test-loss:eval\"][epoch+1] = loss_test_eval\n            #self.outdata[\"train-loss:eval\"][epoch+1] = loss_train_eval\n            print(\"=================================== Epoch(%d) Custom Metrics: End ===================================\" % (epoch+1))\n\nif __name__ == '__main__':\n    study_csv_path_training = inputdir+'/train_study_level.csv'\n    image_csv_path_training = inputdir+'/train_image_level.csv'\n    size_csv_path_training = imagedir+'/size.csv'\n    study_columns = ['id','Negative','Typical','Indeterminate','Atypical']\n    \n    padding_value_xy = 0.0\n    padding_value_wh = 0.0\n    \n    epochs = 45\n    validation_split = 0.2\n    steps_per_execution = 1\n    validation_freq = 1\n    fit_verbose = 1\n\n    threshold_wh=1e-6\n\n    include_box_outputs = False\n    num_hidden = [256,128,64]   \n    \n    img_x = 64\n    img_y = 64\n    batch_size = 16\n    image_format = 'jpg'\n\n    pre_model_fn = VGG16\n    pre_model_name = 'VGG16_with_study_labels_only'\n\n    model_labels = [pre_model_name + ' transfer model with hidden layers = 2',pre_model_name + ' transfer model with hidden layers = 3',pre_model_name + ' transfer model with hidden layers = 4']\n    #model_labels = [pre_model_name + ' transfer model with hidden layers = 1']\n\n    print(\"\\n=============Started: Reading CSV files ==============\")\n    df_train = read_dataset_csv(study_csv_path_training,image_csv_path_training,size_csv_path_training,study_columns,padding_value_xy,padding_value_wh,image_format)\n    print(\"============Finished: Reading CSV files ==============\")\n\n    max_boxes = int(df_train['ImageBoxCount'].max())\n\n    y_col = study_columns[1:]\n    loss_weights = np.ones(len(y_col))\n\n    if include_box_outputs:\n        for i in range(max_boxes):\n            y_col.append('X'+str(i+1))\n            y_col.append('Y'+str(i+1))\n            y_col.append('W'+str(i+1))\n            y_col.append('H'+str(i+1))\n\n        for i in range(max_boxes):\n            idx = 4*(i+1)\n            if i > 0:\n                loss_weights[idx:idx+4] = 1/(i+1)\n            else:\n                loss_weights[idx:idx+4] = 1\n\n    num_outputs = len(y_col)\n\n    base_model = pre_model_fn(weights=\"imagenet\",include_top=False,input_shape=(img_x,img_y,3))\n    base_model.trainable=False\n\n    title = \"Pre-Trained Base CNN(%s)\" % pre_model_name\n    #writeModelSummary(base_model,projdir.replace('\\\\', '/') + '/results/' + pre_model_name + '_pre_model_summary.txt',title)\n\n    models = dict()\n    for idx in range(len(model_labels)):\n        print(\"\\n=============Started: Building %s ==============\" % model_labels[idx])\n        pre_model = pre_model_fn(weights=\"imagenet\",include_top=False,input_shape=(img_x,img_y,3))\n        pre_model.trainable=False\n        models[model_labels[idx]] = Sequential()\n        models[model_labels[idx]].add(pre_model)\n        models[model_labels[idx]].add(Flatten())\n        models[model_labels[idx]].add(Dense(16,activation='relu'))\n        for h in range(idx+1):\n            models[model_labels[idx]].add(Dense(num_hidden[h],activation='relu'))\n        models[model_labels[idx]].add(Dense(num_outputs))\n\n        title = pre_model_name + \"_with_\" + str(idx+1) + \"_hidden_layers\"\n        #writeModelSummary(models[model_labels[idx]],projdir.replace('\\\\', '/') + '/results/' + title + '_summary.txt',title)\n        models[model_labels[idx]].compile(optimizer=optimizers.RMSprop(),loss=losses.MeanSquaredError(),loss_weights=loss_weights)\n        #models[model_labels[idx]].compile(optimizer=optimizers.RMSprop(),loss=losses.CategoricalCrossentropy(), metrics=[metrics.CategoricalAccuracy()],steps_per_execution=steps_per_execution)\n        print(\"============Finished: Building %s ==============\" % model_labels[idx])\n\n    print(\"\\n=============Started: Reading training images ==============\")\n    generator_train = get_image_generator(df_train,imagedir + '/train','fname',y_col,img_x,img_y,batch_size,validation_split,'training')\n    trainX,trainY = iterator_to_numpy(df_train,imagedir + '/train','fname',y_col,img_x,img_y,batch_size,validation_split,'training',generator_train)\n    print(\"============Finished: Reading training images ==============\")\n\n    print(\"\\n=============Started: Reading test images ==============\")\n    generator_test = get_image_generator(df_train,imagedir + '/train','fname',y_col,img_x,img_y,batch_size,validation_split,'validation')\n    testX,testY = iterator_to_numpy(df_train,imagedir + '/train','fname',y_col,img_x,img_y,batch_size,validation_split,'validation',generator_test)\n    print(\"============Finished: Reading test images ==============\")\n\n    data = dict()\n    for label in model_labels:\n        data[label] = dict()\n        data[label][\"test-accuracy:study\"] = dict()\n        data[label][\"train-accuracy:study\"] = dict()\n        data[label][\"test-loss:study\"] = dict()\n        data[label][\"train-loss:study\"] = dict()\n\n        data[label][\"test-accuracy:opacity\"] = dict()\n        data[label][\"train-accuracy:opacity\"] = dict()\n        data[label][\"test-loss:opacity\"] = dict()\n        data[label][\"train-loss:opacity\"] = dict()\n    \n        data[label][\"test-accuracy:bbox\"] = dict()\n        data[label][\"train-accuracy:bbox\"] = dict()\n        data[label][\"test-loss:bbox\"] = dict()\n        data[label][\"train-loss:bbox\"] = dict()\n\n        data[label][\"test-accuracy:eval\"] = dict()\n        data[label][\"train-accuracy:eval\"] = dict()\n        data[label][\"test-loss:eval\"] = dict()\n        data[label][\"train-loss:eval\"] = dict()\n\n    for idx in range(len(model_labels)):\n        rlr = ReduceLROnPlateau(monitor = 'loss', factor = 0.1, patience = 2, verbose = 1,min_delta = 1e-4, min_lr = 1e-6, mode = 'min')\n        es = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min',restore_best_weights = True, verbose = 1)\n        ckp = ModelCheckpoint('model.h5',monitor = 'val_loss',verbose = 0, save_best_only = True, mode = 'min')\n        metricsCallBack = TestCallback(testX,testY,trainX,trainY,data[model_labels[idx]],validation_freq,batch_size,threshold_wh)\n        #metricsCallBack = TestCallback(None,None,None,None,data[model_labels[idx]],validation_freq,batch_size,threshold_wh,generator_train,generator_test)\n\n        print(\"\\n=============Started: Training %s ==============\" % model_labels[idx])\n        models[model_labels[idx]].fit(trainX, trainY, epochs=epochs,batch_size=batch_size,callbacks=[rlr,es, metricsCallBack],verbose=fit_verbose,shuffle=True)\n        #models[model_labels[idx]].fit(generator_train, epochs=epochs,batch_size=batch_size,callbacks=[rlr,metricsCallBack],verbose=fit_verbose,shuffle=True)\n        print(\"============Finished: Training %s ==============\" % model_labels[idx])\n\n        print(\"\\n========Started: Evaluating Trained %s =========\" % model_labels[idx])\n        title = model_labels[idx]\n        show_model_metrics(models[model_labels[idx]],testX,testY,trainX,trainY,title,study_columns[1:],batch_size)\n        #show_model_metrics(models[model_labels[idx]],testX,testY,trainX,trainY,title,study_columns[1:],batch_size,generator_train,generator_test)\n\n        plot_per_epoch_data(data[model_labels[idx]],model_labels[idx])\n        print(\"========Finished: Evaluating Trained %s ========\" % model_labels[idx])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-11T18:45:53.938884Z","iopub.execute_input":"2021-06-11T18:45:53.939319Z","iopub.status.idle":"2021-06-11T22:50:09.983462Z","shell.execute_reply.started":"2021-06-11T18:45:53.939280Z","shell.execute_reply":"2021-06-11T22:50:09.982399Z"},"trusted":true},"execution_count":null,"outputs":[]}]}