{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nimport glob\nfrom tqdm.notebook import tqdm\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom tensorflow.keras.preprocessing.image import img_to_array\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.optimizers import RMSprop , SGD\nfrom keras import backend as K\nfrom tensorflow.keras.utils import plot_model\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping , ModelCheckpoint , ReduceLROnPlateau\nfrom tensorflow.keras.layers.experimental.preprocessing import RandomFlip,RandomRotation\nfrom tensorflow.keras.layers import Dense , Input , Conv2D  ,MaxPooling2D ,Flatten , Lambda ,Average, UpSampling2D ,Conv2DTranspose ,Reshape\nfrom tensorflow.keras.applications.vgg19 import VGG19\nfrom keras.utils import to_categorical\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = train[train.target == 0].image_name.values\npaths.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data_augmentation\n\ndata_augmentation = tf.keras.Sequential([\n  RandomFlip(\"horizontal_and_vertical\"),\n  RandomRotation(0.2),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# AE encoder \n'''\ninput shape = (256,256,3)\n'''\n\ninput_shape = (256,256,3)\n\ninput_encoder = Input(input_shape,name = 'encoder_input')\nencoder = Conv2D(16,(3,3),activation = 'relu',name = 'encoder_layer1')(input_encoder)\nencoder = Conv2D(16,(3,3),activation = 'relu',name = 'encoder_layer1_1')(encoder)\nencoder = Conv2D(32,(3,3),activation = 'relu',name = 'encoder_layer1_2')(encoder)\nencoder = MaxPooling2D(2,2 , name = 'encoder_layer2')(encoder)\nencoder = Conv2D(32,(3,3),activation = 'relu',name = 'encoder_layer3')(encoder)\nencoder = Conv2D(32,(3,3),activation = 'relu',name = 'encoder_layer3_1')(encoder)\nencoder = Conv2D(32,(3,3),activation = 'relu',name = 'encoder_layer3_2')(encoder)\nencoder = MaxPooling2D(3,3 , name = 'encoder_layer4')(encoder)\nencoder = Conv2D(64,(3,3),activation = 'relu',name = 'encoder_layer5')(encoder)\nencoder = Conv2D(64,(3,3),activation = 'relu',name = 'encoder_layer5_1')(encoder)\nencoder = Conv2D(64,(4,4),activation = 'relu',name = 'encoder_layer5_2')(encoder)\nencoder = Conv2D(64,(4,4),activation = 'relu',name = 'encoder_layer5_3')(encoder)\nencoder = MaxPooling2D(3,3 , name = 'encoder_layer6')(encoder)\nencoder = Flatten(name = 'encoder_layer7')(encoder)\nencoder = Dense(256 , activation = 'relu', name = 'encoder_layer8')(encoder)\n\nEncoder = Model(inputs= [input_encoder], outputs=[encoder],name = 'Encoder')\nprint(Encoder.summary())\nplot_model(Encoder, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#AE decoder\n\n'''\ninput shape = (256)\n'''\n\ninput_shape = 256\n\ninput_decoder = Input(input_shape,name = 'decoder_input')\ndecoder = Dense((5184),name = 'decoder_layer1_1')(input_decoder)\ndecoder = Reshape((9, 9, 64),name = 'decoder_layer1_2')(decoder)\ndecoder = UpSampling2D((3,3),name = 'decoder_layer1_3')(decoder)\ndecoder = Conv2DTranspose(64, (3,3), activation='relu' , name = 'decoder_layer1_4')(decoder)\ndecoder = Conv2DTranspose(64, (4,4), activation='relu' , name = 'decoder_layer2')(decoder)\ndecoder = Conv2DTranspose(64, (4,4), activation='relu' , name = 'decoder_layer2_1')(decoder)\ndecoder = Conv2DTranspose(64, (3,3), activation='relu' , name = 'decoder_layer2_2')(decoder)\ndecoder = Conv2DTranspose(64, (3,3), activation='relu' , name = 'decoder_layer2_3')(decoder)\ndecoder = UpSampling2D((3,3),name = 'decoder_layer3')(decoder)\ndecoder = Conv2DTranspose(32, (3,3), activation='relu' , name = 'decoder_layer3_1')(decoder)\ndecoder = Conv2DTranspose(32, (3,3), activation='relu' , name = 'decoder_layer4')(decoder)\ndecoder = Conv2DTranspose(32, (3,3), activation='relu' , name = 'decoder_layer4_1')(decoder)\ndecoder = Conv2DTranspose(32, (3,3), activation='relu' , name = 'decoder_layer4_2')(decoder)\ndecoder = UpSampling2D((2,2),name = 'decoder_layer5')(decoder)\ndecoder = Conv2DTranspose(32, (3,3), activation='relu' , name = 'decoder_layer6')(decoder)\ndecoder = Conv2DTranspose(16, (3,3), activation='relu' , name = 'decoder_layer6_1')(decoder)\ndecoder = Conv2DTranspose(16, (3,3), activation='relu' , name = 'decoder_layer6_2')(decoder)\noutput_decoder = Conv2D(3,(1,1), name = 'decoder_layer9' , activation= 'relu')(decoder)\n\nDecoder = Model(inputs= [input_decoder], outputs=[output_decoder],name = 'Decoder')\nprint(Decoder.summary())\nplot_model(Decoder, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AE_input = Input((256,256,3),name = 'AE_input')\nAE_latent = Encoder(AE_input)\nAE_output = Decoder(AE_latent)\nAE = Model(inputs= [AE_input], outputs=[AE_output],name = 'AE')\n\nprint(AE.summary())\nplot_model(AE, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndefine perceptual_loss\n'''\n\nselected_layers = ['block1_conv1', 'block2_conv2',\"block3_conv1\",\"block3_conv3\" ,'block4_conv1',\n                   'block4_conv3','block5_conv1','block5_conv2','block5_conv3','block5_conv4']\n\nselected_layer_weights = [1.0, 1.0 , 2.0 , 2.0 ,4.0,\n                         4.0 , 8.0 ,8.0 ,16.0, 32.0]\n\nvgg = VGG19(weights='imagenet', include_top=False, input_shape=(256,256,3))\nvgg.trainable = False\noutputs = [vgg.get_layer(l).output for l in selected_layers]\nmodel = Model(vgg.input, outputs)\n\n@tf.function\ndef perceptual_loss(input_image , reconstruct_image):\n    h1_list = model(input_image)\n    h2_list = model(reconstruct_image)\n\n    rc_loss = 0.0\n\n    img = K.batch_flatten(input_image)\n    r_img = K.batch_flatten(reconstruct_image)\n    r_error =  K.sum(K.abs(img - r_img), axis=-1) /(256 * 256)\n\n    for h1, h2, weight in zip(h1_list, h2_list, selected_layer_weights):\n\n        h1 = K.batch_flatten(h1)\n        h2 = K.batch_flatten(h2)\n        rc_loss = rc_loss + weight * K.sum(K.abs(h1 - h2), axis=-1)\n\n    rc_loss = (rc_loss / (sum(selected_layer_weights)))\n    error = (rc_loss + r_error)/2\n\n    return error ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rmsprop = RMSprop(learning_rate=0.0001)\nAE.compile(loss= perceptual_loss, optimizer= rmsprop)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor=\"val_loss\",\n                   patience=15)\n\nrs = ReduceLROnPlateau(monitor=\"val_loss\",\n                  factor=0.1,\n                  patience=5,\n                  verbose=1,\n                  mode=\"auto\")\n\ncheck = ModelCheckpoint('check.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root = '../input/siim-isic-melanoma-classification/jpeg/train/'\n\nimages = []\n\nfor path in tqdm(paths[:2500]):\n    path = root + path + '.jpg'\n    img = load_img(path,target_size=(256,256))\n    img = img_to_array(img)\n    img = img.astype(np.float32)\n    img = (img)/255.0\n    img = np.asarray(img)\n    img = img.reshape(1,256,256,3)\n    img = data_augmentation(img)\n    images.append(img[0])\nimages = np.asarray(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AE.fit(x=images,y=images,\n    epochs=60,\n    verbose = 1,\n    batch_size = 8,\n    validation_split = 0.3,\n    callbacks = [es,rs,check])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root = '../input/siim-isic-melanoma-classification/jpeg/train/'\n\nimages = []\n\nfor path in tqdm(paths[2500:5000]):\n    path = root + path + '.jpg'\n    img = load_img(path,target_size=(256,256))\n    img = img_to_array(img)\n    img = img.astype(np.float32)\n    img = (img)/255.0\n    img = np.asarray(img)\n    img = img.reshape(1,256,256,3)\n    img = data_augmentation(img)\n    images.append(img[0])\nimages = np.asarray(images)\n\nprint(images.shape)\n\nAE.fit(x=images,y=images,\n    epochs=60,\n    verbose = 1,\n    batch_size = 8,\n    validation_split = 0.3,\n    callbacks = [es,rs,check])\n\nAE.save_weights('AE_weights.h5')\n\ndel images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def error(img,r_img):\n    img = K.batch_flatten(img)\n    r_img = K.batch_flatten(r_img)\n    r_error =  K.sum(K.abs(img - r_img), axis=-1)\n    return r_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = train[train.target == 1].image_name.values\nroot = '../input/siim-isic-melanoma-classification/jpeg/train/'\n\nerrors = []\ntargets = []\n\nfor path in tqdm(paths):\n    label = train[train.image_name == path].target.values[0]\n    targets.append(label)\n    path = root + path + '.jpg'\n    img = load_img(path,target_size=(256,256))\n    img = img_to_array(img)\n    img = img.astype(np.float32)\n    img = (img)/255.0\n    img = np.asarray(img)\n    img = img.reshape(1,256,256,3)\n    reconstruct = AE(img)\n    r_error = error(img[0],reconstruct[0])\n    errors.append(r_error) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errors[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = train[train.target == 0].image_name.values\npaths = paths[:584]\nroot = '../input/siim-isic-melanoma-classification/jpeg/train/'\n\nfor path in tqdm(paths):\n    label = train[train.image_name == path].target.values[0]\n    targets.append(label)\n    path = root + path + '.jpg'\n    img = load_img(path,target_size=(256,256))\n    img = img_to_array(img)\n    img = img.astype(np.float32)\n    img = (img)/255.0\n    img = np.asarray(img)\n    img = img.reshape(1,256,256,3)\n    reconstruct = AE(img)\n    r_error = error(img[0],reconstruct[0])\n    errors.append(r_error) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errors = np.asarray(errors)\ntargets = np.asarray(targets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errors.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#target_SVM = targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#targets = to_categorical(targets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# input_layer = Input(shape=(256), name =\"input\")\n\n# layer1 = Dense(256 ,activation='relu' , name=\"layer1-1\")(input_layer)\n# layer1 = Dense(128 ,activation='relu' , name=\"layer2-1\")(layer1)\n# layer1 = Dense(64 ,activation='relu' , name=\"layer4-1\")(layer1)\n# layer1 = Dense(32 ,activation='relu' , name=\"layer5-1\")(layer1)\n# output = Dense(2 ,activation='softmax' , name=\"output1\")(layer1)\n\n# model1 = Model(inputs=input_layer, outputs=output , name = \"Classifier1\")\n# ######################\n# layer2 = Dense(256 ,activation='relu' , name=\"layer1-2\")(input_layer)\n# layer2 = Dense(64 ,activation='relu' , name=\"layer2-2\")(layer2)\n# layer2 = Dense(64 ,activation='relu' , name=\"layer3-2\")(layer2)\n# layer2 = Dense(16 ,activation='relu' , name=\"layer4-2\")(layer2)\n# layer2 = Dense(8 ,activation='relu' , name=\"layer5-2\")(layer2)\n# output = Dense(2 ,activation='softmax' , name=\"output2\")(layer2)\n\n# model2 = Model(inputs=input_layer, outputs=output , name = \"Classifier2\")\n# ######################\n# layer3 = Dense(256 ,activation='relu' , name=\"layer1-3\")(input_layer)\n# layer3 = Dense(256 ,activation='relu' , name=\"layer2-3\")(layer3)\n# layer3 = Dense(64 ,activation='relu' , name=\"layer3-3\")(layer3)\n# layer3 = Dense(16 ,activation='relu' , name=\"layer4-3\")(layer3)\n# output = Dense(2 ,activation='softmax' , name=\"output3\")(layer3)\n\n# model3 = Model(inputs=input_layer, outputs=output , name = \"Classifier3\")\n# ######################\n\n# print(model1.summary())\n# print(model2.summary())\n# print(model3.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# es = EarlyStopping(monitor=\"val_loss\",\n#                    patience=30)\n\n# rs = ReduceLROnPlateau(monitor=\"val_loss\",\n#                   factor=0.1,\n#                   patience=10,\n#                   verbose=1,\n#                   mode=\"auto\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# rmsprop = RMSprop(learning_rate=0.0001)\n# sgd = SGD(learning_rate=0.0001,momentum=0.99)\n# model1.compile(loss= 'binary_crossentropy', metrics= [tf.keras.metrics.Accuracy(),tf.keras.metrics.AUC()], optimizer= rmsprop)\n# model2.compile(loss= 'binary_crossentropy', metrics= [tf.keras.metrics.Accuracy(),tf.keras.metrics.AUC()], optimizer= sgd)\n# model3.compile(loss= 'binary_crossentropy', metrics= [tf.keras.metrics.Accuracy(),tf.keras.metrics.AUC()], optimizer= rmsprop)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model1.fit(x=errors,y=targets,\n#     epochs=150,\n#     verbose = 1,\n#     batch_size = 32,\n#     validation_split = 0.3,\n#     callbacks = [es,rs])\n\n# model1.save_weights('Classifier1_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model2.fit(x=errors,y=targets,\n#     epochs=150,\n#     verbose = 1,\n#     batch_size = 64,\n#     validation_split = 0.3,\n#     callbacks = [es,rs])\n\n# model2.save_weights('Classifier2_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model3.fit(x=errors,y=targets,\n#     epochs=150,\n#     verbose = 1,\n#     batch_size = 16,\n#     validation_split = 0.3,\n#     callbacks = [es,rs])\n\n# model3.save_weights('Classifier3_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.svm import SVC\n# clf = SVC()\n# clf.fit(errors, target_SVM)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameters = {\n    'n_estimators': [1,2,4,8,16,32,64,128,256]}\nclassifier = GridSearchCV(AdaBoostClassifier(), parameters, scoring='accuracy', n_jobs= 4, cv=5)\nclassifier.fit(errors, targets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AdaBoost = classifier.best_estimator_\nprint (classifier.best_score_, classifier.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del targets\ndel errors\n#del target_SVM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = test.image_name.values\nroot = '../input/siim-isic-melanoma-classification/jpeg/test/'\n\n# preds1 = []\n# preds2 = []\n# preds3 = []\npreds4 = []\n\nfor path in tqdm(paths):\n    path = root + path + '.jpg'\n    img = load_img(path,target_size=(256,256))\n    img = img_to_array(img)\n    img = img.astype(np.float32)\n    img = (img)/255.0\n    img = np.asarray(img)\n    img = img.reshape(1,256,256,3)\n    reconstruct = AE(img)\n    r_error = error(img[0],reconstruct[0]).numpy()\n    r_error = r_error.reshape(1,256)\n    \n#     pred1 = model1(r_error)\n#     pred1 = np.argmax(pred1, axis=1)\n#     preds1.append(pred1)\n    \n#     pred2 = model2(r_error)\n#     pred2 = np.argmax(pred2, axis=1)\n#     preds2.append(pred2)\n    \n#     pred3 = model3(r_error)\n#     pred3 = np.argmax(pred3, axis=1)\n#     preds3.append(pred3)\n    \n    pred4 = AdaBoost.predict(r_error)\n    preds4.append(pred4)\n    \n#preds1 = np.asarray(preds1)\n# preds2 = np.asarray(preds2)\n# preds3 = np.asarray(preds3)\npreds4 = np.asarray(preds4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds1[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds2[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds3[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds4[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds2.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds3.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds4.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub = pd.DataFrame({'image_name':paths , 'target1':preds1[:,0],'target2':preds2[:,0],'target3':preds3[:,0],'target4':preds4[:,0]})\nsub = pd.DataFrame({'image_name':paths,'target':preds4[:,0]})\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub['target'] = (round((sub['target1'] + sub['target2'] + sub['target3'] + 3*sub['target4'])/6,0))\n#sub['target'] = sub['target'].astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub = sub[['image_name','target']]\n#sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}