{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport numpy as np\nimport random\nimport time\nimport os\n#os.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,40).__str__()\nimport cv2\nfrom tqdm import tqdm\n\nimport tensorflow as tf\nfrom tensorflow.python.keras import Sequential\nfrom tensorflow.keras import layers, optimizers\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.initializers import glorot_uniform\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom IPython.display import display\nfrom tensorflow.keras import backend as K\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom sklearn.model_selection import train_test_split\nfrom keras import optimizers\n#from sklearn.metrics import classification_report, confusion_matrix\nimport sklearn\nimport seaborn as sn\nfrom keras.callbacks import CSVLogger, LambdaCallback\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:21.975047Z","iopub.execute_input":"2021-06-29T16:28:21.975328Z","iopub.status.idle":"2021-06-29T16:28:27.561430Z","shell.execute_reply.started":"2021-06-29T16:28:21.975266Z","shell.execute_reply":"2021-06-29T16:28:27.560364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = '../input/siim-isic-melanoma-classification/jpeg/' ","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:27.564259Z","iopub.execute_input":"2021-06-29T16:28:27.564610Z","iopub.status.idle":"2021-06-29T16:28:27.573422Z","shell.execute_reply.started":"2021-06-29T16:28:27.564573Z","shell.execute_reply":"2021-06-29T16:28:27.572420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = \"../input/siim-isic-melanoma-classification/train.csv\"\ndf_train = pd.read_csv(train_csv) # , delimiter = \" \", header=None\n#train_df_original.columns = ['Column_1', 'filename', 'label', 'Column_2']\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:27.578522Z","iopub.execute_input":"2021-06-29T16:28:27.578926Z","iopub.status.idle":"2021-06-29T16:28:27.683352Z","shell.execute_reply.started":"2021-06-29T16:28:27.578889Z","shell.execute_reply":"2021-06-29T16:28:27.682556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train['target'] = df_train['target'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:27.684842Z","iopub.execute_input":"2021-06-29T16:28:27.685323Z","iopub.status.idle":"2021-06-29T16:28:27.689361Z","shell.execute_reply.started":"2021-06-29T16:28:27.685282Z","shell.execute_reply":"2021-06-29T16:28:27.688310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_full_path = df_train.copy()\n\n#'''\nfor i in range(df_train.shape[0]):\n    filename = df_train['image_name'][i]\n    file_full_path = f\"../input/siim-isic-melanoma-classification/jpeg/train/{filename}.jpg\"\n    df_train_full_path['image_name'][i] = file_full_path\n#'''\n    \nprint(f\"df_train_full_path['image_name'][0] = {df_train_full_path['image_name'][0]}\")\ndf_train_full_path.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:27.690859Z","iopub.execute_input":"2021-06-29T16:28:27.691511Z","iopub.status.idle":"2021-06-29T16:28:35.179923Z","shell.execute_reply.started":"2021-06-29T16:28:27.691470Z","shell.execute_reply":"2021-06-29T16:28:35.179077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_full_path['target'] = df_train_full_path['target'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:35.181285Z","iopub.execute_input":"2021-06-29T16:28:35.181644Z","iopub.status.idle":"2021-06-29T16:28:35.227334Z","shell.execute_reply.started":"2021-06-29T16:28:35.181592Z","shell.execute_reply":"2021-06-29T16:28:35.226564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = \"../input/siim-isic-melanoma-classification/test.csv\"\ndf_test = pd.read_csv(test_csv) # , delimiter = \" \", header=None\n#train_df_original.columns = ['Column_1', 'filename', 'label', 'Column_2']\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:35.228562Z","iopub.execute_input":"2021-06-29T16:28:35.228966Z","iopub.status.idle":"2021-06-29T16:28:35.267987Z","shell.execute_reply.started":"2021-06-29T16:28:35.228928Z","shell.execute_reply":"2021-06-29T16:28:35.267073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_full_path = df_test.copy()\n\n#'''\nfor i in range(df_test.shape[0]):\n    filename = df_test['image_name'][i]\n    file_full_path = f\"../input/siim-isic-melanoma-classification/jpeg/test/{filename}.jpg\"\n    df_test_full_path['image_name'][i] = file_full_path\n#'''\n    \nprint(f\"df_test_full_path['image_name'][0] = {df_test_full_path['image_name'][0]}\")\ndf_test_full_path.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:50:45.656383Z","iopub.execute_input":"2021-06-29T18:50:45.656715Z","iopub.status.idle":"2021-06-29T18:50:47.118555Z","shell.execute_reply.started":"2021-06-29T18:50:45.656684Z","shell.execute_reply":"2021-06-29T18:50:47.117487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_classes = np.unique(df_train_full_path.target.values)\nlist_classes","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:36.598488Z","iopub.execute_input":"2021-06-29T16:28:36.598841Z","iopub.status.idle":"2021-06-29T16:28:36.639161Z","shell.execute_reply.started":"2021-06-29T16:28:36.598806Z","shell.execute_reply":"2021-06-29T16:28:36.638253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"work_dir = \"./\"\ncolor_type = 'rgb' # rgb, grayscale\nBATCH_SIZE = 32 #16","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:36.640309Z","iopub.execute_input":"2021-06-29T16:28:36.640650Z","iopub.status.idle":"2021-06-29T16:28:36.644375Z","shell.execute_reply.started":"2021-06-29T16:28:36.640602Z","shell.execute_reply":"2021-06-29T16:28:36.643520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255,\n                                   width_shift_range=0.2,\n                                   height_shift_range=0.2,\n                                   zoom_range=0.2,\n                                   validation_split=0.2\n                                   )\n#val_datagen = ImageDataGenerator(rescale=1./255)\ntest_datagen = ImageDataGenerator(rescale=1./255)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:51:09.221904Z","iopub.execute_input":"2021-06-29T18:51:09.222237Z","iopub.status.idle":"2021-06-29T18:51:09.226529Z","shell.execute_reply.started":"2021-06-29T18:51:09.222208Z","shell.execute_reply":"2021-06-29T18:51:09.225713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = train_datagen.flow_from_dataframe(\n        df_train_full_path,\n        directory=None,\n        x_col='image_name', \n        y_col='target',\n        #target_size=(800, 804),  # target images are automatically resized to (256, 256)\n        batch_size=BATCH_SIZE,\n        shuffle = True,\n        color_mode=color_type, # grayscale, rgb\n        class_mode='categorical', # categorical, raw \n        subset='training'\n        )","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:28:36.655514Z","iopub.execute_input":"2021-06-29T16:28:36.655938Z","iopub.status.idle":"2021-06-29T16:30:04.663201Z","shell.execute_reply.started":"2021-06-29T16:28:36.655903Z","shell.execute_reply":"2021-06-29T16:30:04.662356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = len(train_generator.class_indices)\ntotal_train_data = train_generator.samples\n\nprint(f\"total_train_data = {total_train_data}\")\nprint(f\"train_generator.image_shape = {train_generator.image_shape}\")\nprint(f\"num_classes = {num_classes}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:04.664399Z","iopub.execute_input":"2021-06-29T16:30:04.664761Z","iopub.status.idle":"2021-06-29T16:30:04.673893Z","shell.execute_reply.started":"2021-06-29T16:30:04.664722Z","shell.execute_reply":"2021-06-29T16:30:04.669695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_generator = train_datagen.flow_from_dataframe(\n        df_train_full_path,\n        directory=None,\n        x_col='image_name', \n        y_col='target',\n        #target_size=(800, 804),  # target images are automatically resized to (256, 256)\n        batch_size=BATCH_SIZE,\n        shuffle = True,\n        color_mode=color_type, # grayscale, rgb\n        class_mode='categorical', # categorical, raw\n        subset='validation'\n        )","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:04.675486Z","iopub.execute_input":"2021-06-29T16:30:04.675926Z","iopub.status.idle":"2021-06-29T16:30:15.254082Z","shell.execute_reply.started":"2021-06-29T16:30:04.675890Z","shell.execute_reply":"2021-06-29T16:30:15.253201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_val_data = val_generator.samples\nprint(f\"total_val_data = {total_val_data}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.255306Z","iopub.execute_input":"2021-06-29T16:30:15.255674Z","iopub.status.idle":"2021-06-29T16:30:15.262545Z","shell.execute_reply.started":"2021-06-29T16:30:15.255635Z","shell.execute_reply":"2021-06-29T16:30:15.261580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:38:36.135185Z","iopub.execute_input":"2021-06-29T18:38:36.135531Z","iopub.status.idle":"2021-06-29T18:38:36.140060Z","shell.execute_reply.started":"2021-06-29T18:38:36.135499Z","shell.execute_reply":"2021-06-29T18:38:36.138835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator = test_datagen.flow_from_dataframe(\n        df_test_full_path,\n        directory=None,\n        x_col='image_name', \n        #y_col='target',\n        batch_size=1,\n        shuffle = False,\n        color_mode=color_type, # grayscale, rgb\n        class_mode=None # categorical, raw\n        )","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:51:18.889064Z","iopub.execute_input":"2021-06-29T18:51:18.889388Z","iopub.status.idle":"2021-06-29T18:51:25.340379Z","shell.execute_reply.started":"2021-06-29T18:51:18.889357Z","shell.execute_reply":"2021-06-29T18:51:25.339503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_test_data = test_generator.samples\nprint(f\"total_test_data = {total_test_data}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:54:06.444395Z","iopub.execute_input":"2021-06-29T18:54:06.444765Z","iopub.status.idle":"2021-06-29T18:54:06.451889Z","shell.execute_reply.started":"2021-06-29T18:54:06.444733Z","shell.execute_reply":"2021-06-29T18:54:06.450969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DenseNet121 ResNet101 ResNet50 DenseNet201 InceptionV3 Xception NASNetLarge ResNet152V2 InceptionResNetV2 EfficientNetB7\nimpl_type = \"TransferLearning3D.DenseNet201.\" # TransferLearning3D \ndataset = f\"SIIM_ISIC_Melanoma_Classification_Kaggle.DataAug.{color_type}.{train_generator.image_shape[1]}p.DataFlow\" # +str(img_size)+\"p\"\ndataset","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.263792Z","iopub.execute_input":"2021-06-29T16:30:15.264422Z","iopub.status.idle":"2021-06-29T16:30:15.273741Z","shell.execute_reply.started":"2021-06-29T16:30:15.264384Z","shell.execute_reply":"2021-06-29T16:30:15.272714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'''\ncount_no_improvement = 0\nepoch_initial = True\n#'''","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.275027Z","iopub.execute_input":"2021-06-29T16:30:15.275373Z","iopub.status.idle":"2021-06-29T16:30:15.281979Z","shell.execute_reply.started":"2021-06-29T16:30:15.275338Z","shell.execute_reply":"2021-06-29T16:30:15.281134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NUM_NEURONS = 16\n#NUM_LAYERS = 3\n#BATCH_SIZE = 16 # 10\nNUM_EPOCHS = 300 # 300\nepochs_completed = 0\nLEARNING_RATE = 0.00001\nEPSILON = 1e-4\nearly_stop_after_epochs = 5\nDROPOUT = 0.5 # 0.5 0.0\npad = 0\n\nLOSS = 'categorical_crossentropy'\nACTIVATION_FUNCTION = 'elu' # relu sigmoid elu\nFINAL_ACTIVATION_FUNCTION = 'softmax'\nvalidation_split = 0.1\nkernel_size=(1,1)\npointTrainableAfter = \"allDefault\" # \"allDefault\" 160 170\nOPTIMIZER = \"Adam\" # Adam SGD RMSProp\ninit_weights = \"imagenet\" # \"imagenet\" None\nmodelExt = \".Dense.2048.2048.2048.2048\" # .Dense.1024.1024.1024.1024 .Dense.128.256.512, .512.512.512 .Dense.512.512.512.512.Res\nl2_val = 0.001\n\n# +\"_kernel\"+str(kernel_size)+\"_lr\"+str(LEARNING_RATE)+\"_batch\"+str(BATCH_SIZE)+\"_epochs\"+str(NUM_EPOCHS)\n#checkpointer_name  = \"weights_\"+dataset+\"_\"+impl_type+\"_nLayers\"+str(NUM_LAYERS)+\"_nNeurons\"+str(NUM_NEURONS)+\".hdf5\"\next = f\".Flatten.l2.{str(l2_val)}.run_2\" # run_1 run_2 .DropAfter .momentum0.9\n#'''\ncheckpointer_name  = \"weights.\"+dataset+\".pad\"+str(pad)+\".\"+impl_type+\".wInit.\"+str(init_weights)+\".TrainableAfter.\"+str(pointTrainableAfter)+\\\n                     modelExt+\".actF.\"+ACTIVATION_FUNCTION+\".opt.\"+OPTIMIZER+\".drop.\"+str(DROPOUT)+\".batch\"+str(BATCH_SIZE)+ext+\".hdf5\"\nlog_name = \"log.\"+checkpointer_name[8:-5]+\".log\"\n\nprint('checkpointer_name =', checkpointer_name)\nprint('log_name =', log_name)\n#'''","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.283189Z","iopub.execute_input":"2021-06-29T16:30:15.283715Z","iopub.status.idle":"2021-06-29T16:30:15.293968Z","shell.execute_reply.started":"2021-06-29T16:30:15.283680Z","shell.execute_reply":"2021-06-29T16:30:15.293048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator.image_shape","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.295200Z","iopub.execute_input":"2021-06-29T16:30:15.295561Z","iopub.status.idle":"2021-06-29T16:30:15.309273Z","shell.execute_reply.started":"2021-06-29T16:30:15.295527Z","shell.execute_reply":"2021-06-29T16:30:15.308535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'''\n#base_model=DenseNet121(weights=None, include_top=False, input_shape=np_train_dataset2.shape[1:]) # `None` (random initialization)\n#base_model=ResNet152V2(weights=None, include_top=False, input_shape=np_train_dataset2.shape[1:])\n# ResNet152V2 ResNet50 ResNet101 ResNet152 DenseNet201 InceptionV3 Xception NASNetLarge 'imagenet' ResNet152V2 DenseNet121\n#inputs = Input(final_train_imageset.shape[1:])\n#x = ZeroPadding2D(padding=(pad,pad))(inputs)\n#base_model=tf.keras.applications.ResNet50(weights=init_weights, include_top=False, input_tensor=x)\nbase_model=tf.keras.applications.DenseNet201(weights=init_weights, include_top=False, input_shape=train_generator.image_shape)\n#base_model=tf.keras.applications.ResNet152V2(weights=init_weights, include_top=False, input_shape=train_generator.image_shape)\nx=base_model.output\n\nx = Flatten()(x)\n\n#'''\nx = Dense(2048, kernel_regularizer=tf.keras.regularizers.l2(l2_val), activation=ACTIVATION_FUNCTION)(x)\n#x_copy = x\nx = Dropout(DROPOUT)(x)\nx = Dense(2048, kernel_regularizer=tf.keras.regularizers.l2(l2_val), activation=ACTIVATION_FUNCTION)(x)\nx = Dropout(DROPOUT)(x)\nx = Dense(2048, kernel_regularizer=tf.keras.regularizers.l2(l2_val), activation=ACTIVATION_FUNCTION)(x)\nx = Dropout(DROPOUT)(x)\nx = Dense(2048, kernel_regularizer=tf.keras.regularizers.l2(l2_val), activation=ACTIVATION_FUNCTION)(x)\nx = Dropout(DROPOUT)(x)\n#x = Add()([x,x_copy])\n#'''\noutputs=Dense(num_classes,activation='softmax')(x)\n\nmodel=Model(inputs=base_model.input,outputs=outputs)\n#model.summary()\n#'''","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:15.310497Z","iopub.execute_input":"2021-06-29T16:30:15.310848Z","iopub.status.idle":"2021-06-29T16:30:24.794012Z","shell.execute_reply.started":"2021-06-29T16:30:15.310815Z","shell.execute_reply":"2021-06-29T16:30:24.793189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntf.keras.utils.plot_model(\n    model, to_file='model.png', show_shapes=True, show_dtype=False,\n    show_layer_names=True, rankdir='TB', expand_nested=True, dpi=64\n)\n#'''","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.795170Z","iopub.execute_input":"2021-06-29T16:30:24.795489Z","iopub.status.idle":"2021-06-29T16:30:24.800369Z","shell.execute_reply.started":"2021-06-29T16:30:24.795457Z","shell.execute_reply":"2021-06-29T16:30:24.799338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_trainable = 0\ncount_non_trainable = 0\n\n#'''\nif pointTrainableAfter == \"allDefault\":\n    for layer in model.layers:\n        layer.trainable=True\n        count_trainable += 1\nelif pointTrainableAfter > 0:\n    for layer in model.layers[:pointTrainableAfter]: # [:-pointTrainableAfter]\n        layer.trainable=False\n        count_non_trainable += 1\n    for layer in model.layers[pointTrainableAfter:]: # [-pointTrainableAfter:]\n        layer.trainable=True\n        count_trainable += 1\n#'''\n\n'''\nfor layer in model.layers:\n    layer.trainable=True\n    count_trainable += 1\n#'''\n\nprint(\"count_non_trainable =\", count_non_trainable)\nprint(\"count_trainable =\", count_trainable)\nprint(\"Total number of layers =\", count_non_trainable+count_trainable)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.801605Z","iopub.execute_input":"2021-06-29T16:30:24.802094Z","iopub.status.idle":"2021-06-29T16:30:24.833581Z","shell.execute_reply.started":"2021-06-29T16:30:24.802059Z","shell.execute_reply":"2021-06-29T16:30:24.832381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"RMSProp\" \"SGD\" \"Adam\" \"Adamax\" \"Adadelta\" \"Adagrad\" \"SGD\"\n#optimizer = tf.keras.optimizers.RMSprop(lr = LEARNING_RATE, epsilon=EPSILON)\n\nif OPTIMIZER == \"RMSProp\":\n    optimizer = tf.keras.optimizers.RMSprop(lr = LEARNING_RATE, epsilon=EPSILON)\nelif OPTIMIZER == \"Adam\":\n    optimizer = tf.keras.optimizers.Adam(lr = LEARNING_RATE, epsilon=EPSILON, beta_1=0.9, beta_2=0.999)\nelif OPTIMIZER == \"Adamax\":\n    optimizer = tf.keras.optimizers.Adamax(lr = LEARNING_RATE, epsilon=EPSILON, beta_1=0.9, beta_2=0.999)\nelif OPTIMIZER == \"Adadelta\":\n    optimizer = tf.keras.optimizers.Adadelta(lr = LEARNING_RATE, epsilon=EPSILON, rho=0.95)\nelif OPTIMIZER == \"Adagrad\":\n    optimizer = tf.keras.optimizers.Adagrad(lr = LEARNING_RATE, epsilon=EPSILON, initial_accumulator_value=0.1)\nelif OPTIMIZER == \"SGD\":\n    optimizer = tf.keras.optimizers.SGD(lr = LEARNING_RATE, momentum=0.9)\n\nmodel.compile(\n    #optimizer=OPTIMIZER,\n    optimizer=optimizer,\n    loss=LOSS,\n    metrics=['accuracy','AUC']\n)\n\nprint(\"OPTIMIZER =\", OPTIMIZER)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.836999Z","iopub.execute_input":"2021-06-29T16:30:24.837363Z","iopub.status.idle":"2021-06-29T16:30:24.869051Z","shell.execute_reply.started":"2021-06-29T16:30:24.837316Z","shell.execute_reply":"2021-06-29T16:30:24.868187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the best model with least validation loss\ncheckpointer = ModelCheckpoint(filepath = work_dir+checkpointer_name, \n                               monitor='val_accuracy',\n                               #monitor='val_loss',\n                               save_weights_only=False,  \n                               mode='auto', \n                               verbose = 0, # 0 = silent, 1 = progress bar, 2 = one line per epoch\n                               save_best_only =False\n                               )\ncheckpointer_best = ModelCheckpoint(filepath = work_dir+\"best_\"+checkpointer_name, \n                                    monitor='val_accuracy',\n                                    #monitor='val_loss', \n                                    save_weights_only=False,\n                                    mode='auto',  \n                                    verbose = 1, \n                                    save_best_only = True\n                                    )\nearly_stopping = EarlyStopping(monitor='loss', patience=early_stop_after_epochs)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.870390Z","iopub.execute_input":"2021-06-29T16:30:24.870773Z","iopub.status.idle":"2021-06-29T16:30:24.877444Z","shell.execute_reply.started":"2021-06-29T16:30:24.870739Z","shell.execute_reply":"2021-06-29T16:30:24.876322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif 'count_no_improvement' not in globals():\n    count_no_improvement = 0\n    print(\"count_no_improvement =\", count_no_improvement)\n#'''\n'''\ncount_no_improvement = 0\nepoch_initial = False\n#'''\nmin_delta = 0.0009\nprint(\"count_no_improvement =\", count_no_improvement)\n\ndef checkBestPerformance(epoch, logs):\n    save_filepath = work_dir+\"best_\"+checkpointer_name\n\n    global epoch_initial\n    if epoch_initial == True:\n        epoch_initial = False\n        model.save(filepath = save_filepath)\n        print(\". Model saved!\")\n\n    elif epoch_initial == False:\n        global count_no_improvement\n\n        log_data = pd.read_csv(work_dir+log_name, sep=',', usecols=['val_loss', 'val_accuracy'], engine='python')\n        min_val_loss = float(str(min(log_data.val_loss.values))[:6])\n        max_val_acc = float(str(max(log_data.val_accuracy.values))[:6])\n\n        current_val_acc = float(str(logs['val_accuracy'])[:6])\n        current_val_loss = float(str(logs['val_loss'])[:6])\n\n        if (current_val_acc > max_val_acc) and (abs(current_val_loss-min_val_loss) >= min_delta):\n            count_no_improvement = 0\n            model.save(filepath = save_filepath)\n            print(\"\\nval_accuracy increased from\",max_val_acc,\" to\",current_val_acc,\"( val_loss =\",current_val_loss,\").\")\n\n        elif (current_val_loss<min_val_loss) and (current_val_acc==max_val_acc):\n            count_no_improvement = 0\n            model.save(filepath = save_filepath)\n            print(\"\\nval_loss decreased to\", current_val_loss, \".\")\n\n        else:\n            count_no_improvement += 1\n            print(\". count_no_improvement =\", count_no_improvement)\n\n        if count_no_improvement >= early_stop_after_epochs:\n            global list_callbacks\n            del list_callbacks, count_no_improvement\n            #print(\"count_no_improvement =\", count_no_improvement, \"... list_callbacks =\", list_callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.878737Z","iopub.execute_input":"2021-06-29T16:30:24.879223Z","iopub.status.idle":"2021-06-29T16:30:24.892410Z","shell.execute_reply.started":"2021-06-29T16:30:24.879187Z","shell.execute_reply":"2021-06-29T16:30:24.891332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs_completed = 0\nlist_callbacks = []\ncsv_logger = CSVLogger(work_dir+log_name, separator=',', append=True)\n\n#if 'list_callbacks' in globals():\n#    del list_callbacks\n\ntry:\n    log_data = pd.read_csv(work_dir+log_name, sep=',', usecols=['epoch'], engine='python')\n    epochs_completed = log_data.shape[0]\n\n    #if epochs_completed > 0:\n    model = load_model(work_dir+checkpointer_name)\n    list_callbacks = [checkpointer, LambdaCallback(on_epoch_end=checkBestPerformance), csv_logger]\n    print(\"epochs_completed =\", epochs_completed)\n\nexcept Exception as error:\n    if epochs_completed == 0:\n        # list_callbacks = [checkpointer, checkpointer_best, csv_logger, early_stopping] \n        list_callbacks = [checkpointer, LambdaCallback(on_epoch_end=checkBestPerformance), csv_logger]\n        print(\"epochs_completed =\", epochs_completed)\n    elif epochs_completed > 0:\n        print(error)\n\nprint('checkpointer_name =', checkpointer_name)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:24.893915Z","iopub.execute_input":"2021-06-29T16:30:24.894329Z","iopub.status.idle":"2021-06-29T16:30:25.123493Z","shell.execute_reply.started":"2021-06-29T16:30:24.894294Z","shell.execute_reply":"2021-06-29T16:30:25.122253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('checkpointer_name =', checkpointer_name)\nprint(\"Previously completed epochs =\", epochs_completed)\nprint(\"count_no_improvement =\", count_no_improvement, \"\\n\")\n\n#'''\ntry:\n    start_time = time.time()\n    history = model.fit(train_generator, \n                        steps_per_epoch=total_train_data // BATCH_SIZE,\n                        shuffle=True, \n                        epochs = NUM_EPOCHS - epochs_completed, \n                        validation_data=val_generator,\n                        validation_steps=total_val_data // BATCH_SIZE,\n                        callbacks=list_callbacks\n                        )\n    elapsed_time = time.time() - start_time \n    print(\"\\nTime elapsed: \", elapsed_time)\n\nexcept Exception as error:\n    print(\"\\nError:\", error)\n#'''","metadata":{"execution":{"iopub.status.busy":"2021-06-29T16:30:25.124556Z","iopub.execute_input":"2021-06-29T16:30:25.124923Z","iopub.status.idle":"2021-06-29T18:23:24.364373Z","shell.execute_reply.started":"2021-06-29T16:30:25.124885Z","shell.execute_reply":"2021-06-29T18:23:24.361123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weights.SIIM_ISIC_Melanoma_Classification_Kaggle.DataAug.rgb.256p.DataFlow.pad0.TransferLearning3D.DenseNet201..wInit.imagenet.TrainableAfter.allDefault.Dense.2048.2048.2048.2048.actF.elu.opt.Adam.drop.0.5.batch16.Flatten.l2.0.001.run_2.hdf5\n","metadata":{"execution":{"iopub.status.busy":"2021-06-29T18:23:24.367383Z","iopub.status.idle":"2021-06-29T18:23:24.369545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = model.predict_generator(test_generator, \n                                  steps = total_test_data, # // BATCH_SIZE, \n                                  verbose=1\n                                 )","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:12:20.698042Z","iopub.execute_input":"2021-06-29T19:12:20.698369Z","iopub.status.idle":"2021-06-29T19:42:23.908983Z","shell.execute_reply.started":"2021-06-29T19:12:20.698338Z","shell.execute_reply":"2021-06-29T19:42:23.908187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict[-300]","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:42:23.911265Z","iopub.execute_input":"2021-06-29T19:42:23.911614Z","iopub.status.idle":"2021-06-29T19:42:23.922038Z","shell.execute_reply.started":"2021-06-29T19:42:23.911577Z","shell.execute_reply":"2021-06-29T19:42:23.921192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict = np.argmax(predict, axis=1)\ny_predict.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:42:23.924388Z","iopub.execute_input":"2021-06-29T19:42:23.924827Z","iopub.status.idle":"2021-06-29T19:42:23.931896Z","shell.execute_reply.started":"2021-06-29T19:42:23.924791Z","shell.execute_reply":"2021-06-29T19:42:23.930999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict[-300]","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:42:23.933343Z","iopub.execute_input":"2021-06-29T19:42:23.933721Z","iopub.status.idle":"2021-06-29T19:42:23.941522Z","shell.execute_reply.started":"2021-06-29T19:42:23.933687Z","shell.execute_reply":"2021-06-29T19:42:23.940574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_predictions_with_image_names = []\nfor image_name,pred in zip(df_test['image_name'].values,y_predict):\n    list_predictions_with_image_names.append([image_name,pred])\nnp_predictions_with_image_names = np.array(list_predictions_with_image_names)\nnp_predictions_with_image_names.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:54:18.464438Z","iopub.execute_input":"2021-06-29T19:54:18.464784Z","iopub.status.idle":"2021-06-29T19:54:18.506920Z","shell.execute_reply.started":"2021-06-29T19:54:18.464752Z","shell.execute_reply":"2021-06-29T19:54:18.505814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict = pd.DataFrame(np_predictions_with_image_names, columns=[\"image_name\",\"target\"])\ndf_predict.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:54:20.871815Z","iopub.execute_input":"2021-06-29T19:54:20.872215Z","iopub.status.idle":"2021-06-29T19:54:20.882478Z","shell.execute_reply.started":"2021-06-29T19:54:20.872182Z","shell.execute_reply":"2021-06-29T19:54:20.881701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T19:54:25.689640Z","iopub.execute_input":"2021-06-29T19:54:25.690027Z","iopub.status.idle":"2021-06-29T19:54:25.716845Z","shell.execute_reply.started":"2021-06-29T19:54:25.689995Z","shell.execute_reply":"2021-06-29T19:54:25.716074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}