{"cells":[{"metadata":{"id":"ksZv9Q1mo1of","colab_type":"code","outputId":"4864eba2-8834-4476-ebe2-a470f979f529","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":true},"cell_type":"code","source":"%reset -f\nimport os\nimport zipfile\nimport hashlib\nfrom six.moves.urllib.error import URLError\nfrom six.moves.urllib.request import urlretrieve\nimport argparse\nfrom tqdm import tqdm\nfrom keras.applications.resnet50 import ResNet50, preprocess_input as preprocess_resnet, decode_predictions\nfrom keras.applications.inception_v3 import InceptionV3, preprocess_input as preprocess_inception\nfrom keras.applications.vgg16 import VGG16, preprocess_input as preprocess_vgg\nfrom keras.preprocessing import image\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D, Concatenate\nimport numpy as np\n\nimport numpy as np\nimport pandas as pd\nimport time, os, glob\nimport cv2\n\nfrom keras.applications.vgg16 import VGG16\nfrom keras.optimizers import SGD, Adam\nfrom keras.layers import GlobalAveragePooling2D\nfrom keras.models import Model\nfrom keras.applications.imagenet_utils import preprocess_input\n\nfrom os.path import isfile, join, abspath, exists, isdir, expanduser\nfrom os import listdir, makedirs, getcwd, remove\n\n# ! ls -la ../input/train_images","execution_count":null,"outputs":[]},{"metadata":{"id":"A9VLquj9o1ok","colab_type":"text"},"cell_type":"markdown","source":"# Data set"},{"metadata":{"id":"xvaJbluJo1ol","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport os\nfrom glob import glob\nfrom matplotlib.pyplot import imshow\nimport numpy as np\nfrom PIL import Image\n\nlabels = pd.read_csv(\"../input/train.csv\")\nprint(len(labels))\nlabels.head()\n\n\ndata_dir = '../input/'\ntrain_dir = data_dir + '/train_images/'\ntest_dir = data_dir + '/test_images/'\n\nimageList = glob(train_dir + '/**/*.png', recursive=True)\nprint ( \"Number of images: {}\". format (len (imageList)))\nfor img in imageList[0:5]:\n    print(img)\n    \n%matplotlib inline\npil_im = Image.open(imageList[0], 'r')\nimshow(np.asarray(pil_im))","execution_count":null,"outputs":[]},{"metadata":{"id":"8q-70_Yio1om","colab_type":"text"},"cell_type":"markdown","source":"# Define pre trained networks"},{"metadata":{"id":"zlAhIZOho1on","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from tqdm import tqdm # Enable progress bar\nfrom keras.applications.resnet50 import ResNet50, preprocess_input, decode_predictions # Load pre-trained model\nfrom keras.models import Model\nfrom keras.preprocessing import image\nfrom keras.layers import Flatten, Input\n\n    \nclass VGGFV:\n    __name__ = \"VGGFV\"\n\n    def __init__(self, feature_layer=\"block5_conv3\", input_size=224,num_class=1,batch_size=32):\n        base_model = VGG16(weights='imagenet', include_top=False,\n                           input_shape=[input_size,input_size,3], classes=num_class)\n                                        \n        x2 = GlobalAveragePooling2D()(base_model.get_layer(\"block2_conv2\").output)  # 128\n        x3 = GlobalAveragePooling2D()(base_model.get_layer(\"block3_conv3\").output)  # 256\n        x4 = GlobalAveragePooling2D()(base_model.get_layer(\"block4_conv3\").output)  # 512\n        x5 = GlobalAveragePooling2D()(base_model.get_layer(\"block5_conv3\").output)  # 512        \n        x = Concatenate()([x2,x3,x4,x5])        \n\n        model = Model(inputs=base_model.input, outputs=x)        \n        optimizer = Adam(lr=0.0001)\n        model.compile(loss='categorical_crossentropy',\n                      optimizer=optimizer,\n                      metrics=['accuracy'])\n        \n        self.model=model    \n        self.batch_size = batch_size\n        self.data_format = K.image_data_format()\n    \n    def predict(self, x):\n        if self.data_format == \"channels_first\":\n            x = x.transpose(0, 3, 1, 2)\n        x = preprocess_vgg(x.astype(K.floatx()))\n        return self.model.predict(x, batch_size=self.batch_size).squeeze()\n    \ndef find_classes_melanoma(fullDir):\n    def invert_dict(d):\n        return dict([(v, k) for k, v in d.iter()])\n\n    df_labels = pd.read_csv(\"../input/train.csv\", sep=',')                \n    class_to_idx={'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}        \n    \n    num_to_class= dict(zip(class_to_idx.values(), class_to_idx.keys()))            \n    classes= ['0', '1', '2', '3', '4']\n\n    print('Sorted Classes: {}'.format(classes))\n    print('class_to_idx: {}'.format(class_to_idx))\n    print('num_to_class: {}'.format(num_to_class))\n\n    train = []\n    for index, row in (df_labels.iterrows()):\n        id = (row['id_code'])\n        # currImage = os.path.join(fullDir, num_to_class[(int(row['melanoma']))] + '/' + id + '.jpg')\n        currImage_on_disk = os.path.join(fullDir, id + '.png')\n        if os.path.isfile(currImage_on_disk):\n            if (int(row['diagnosis'])) == 0:\n                trait = '0'\n            elif (int(row['diagnosis'])) == 1:\n                trait = '1'\n            elif (int(row['diagnosis'])) == 2:\n                trait = '2'\n            elif (int(row['diagnosis'])) == 3:\n                trait = '3'\n            elif (int(row['diagnosis'])) == 4:                \n                trait = '4'\n\n            train.append(['{}'.format(currImage_on_disk), trait, class_to_idx[trait]])\n    df = pd.DataFrame(train, columns=['file', 'category', 'category_id', ])\n#         if os.path.isfile('full_melanoma_labels.csv'):\n#             os.remove('full_melanoma_labels.csv')    \n#         df.to_csv('full_melanoma_labels.csv', index=None)\n    return classes, class_to_idx, num_to_class, df\n\n# def find_classes(fullDir):    \n#     print (\"Full dir:{}\".format(fullDir))\n#     classes = [d for d in os.listdir(fullDir) if os.path.isdir(os.path.join(fullDir, d))]\n#     classes.sort()\n#     class_to_idx = {classes[i]: i for i in range(len(classes))}\n#     num_to_class = dict(zip(range(len(classes)), classes))\n\n#     print('Sorted Classes: {}'.format(classes))\n#     print('class_to_idx: {}'.format(class_to_idx))\n#     print('num_to_class: {}'.format(num_to_class))\n\n#     train = []\n#     for index, label in enumerate(classes):\n#         path = fullDir + label + '/'\n#         for file in listdir(path):\n#             if file.endswith('tif') or file.endswith('jpg') or file.endswith('png'):\n#                 train.append(['{}/{}/{}'.format(fullDir, label, file), label, index])\n\n#     df = pd.DataFrame(train, columns=['file', 'category', 'category_id', ])\n\n#     return classes, class_to_idx, num_to_class, df\n    ","execution_count":null,"outputs":[]},{"metadata":{"id":"tyA80dMho1oo","colab_type":"text"},"cell_type":"markdown","source":"# Test VGG"},{"metadata":{"id":"NRj-wzzNo1oo","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"feature_layer = \"block5_conv3\" # 512\ninput_size=224\n\nmodel = VGGFV (feature_layer, input_size=input_size,num_class=1)\n\ncolumns=['file', 'category_id', 'fv']\nX_df_train=pd.DataFrame(data=np.zeros((0,len(columns))), columns=columns)\nclasses, class_to_idx, num_to_class, df= find_classes_melanoma(train_dir)\nprint (class_to_idx)\n\n# Empty arrays for storing extracted features\nX_train = []; X_test = []\n\n# Extract bottleneck features of photos for traninig    \nfor index, row in tqdm(df.iterrows()):\n    img = image.load_img(row['file'], target_size=(input_size, input_size))\n    x = image.img_to_array(img)\n    x = np.expand_dims(x, axis=0)    \n    vgg_feature_vector = model.predict(x)\n    X_train.append(vgg_feature_vector)\n#     print (vgg_feature_vector)\n    \n    X_df_train = X_df_train.append({ 'file':row['file'], \n                              'category_id':row['category'],                               \n                              'fv':vgg_feature_vector },\n                                 ignore_index=True)\n    \n\n# X_df_train.category_id.astype(int)    \nprint(X_df_train.info())    \n\nX_df_train.to_csv('predictions_df.csv', index = False)\nX_df_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"id":"N9rAj31ao1oq","colab_type":"code","outputId":"a7e3c57a-9a5a-4fb6-fd99-e8cb3ecc824a","colab":{},"trusted":true},"cell_type":"code","source":"import pandas \nX_df_train_SINGLE=X_df_train.copy(deep=True)\nX_df_train_SINGLE.drop('file', axis=1, inplace=True)\nanswers_1_SINGLE = list (X_df_train_SINGLE['category_id'].values)\n# answers_1_SINGLE= map(int, answers_1_SINGLE)\nX_df_train_SINGLE = X_df_train_SINGLE.drop('category_id', axis=1)\nX_df_train_SINGLE=X_df_train_SINGLE.apply(lambda x: pandas.to_numeric(x, errors='ignore'))\n\nX_df_train_SINGLE.head(3)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"e2mlqhLDo1os","colab_type":"code","outputId":"40e7d808-8887-449b-e3e3-8ebf9f4bc598","colab":{},"trusted":true},"cell_type":"code","source":"# Function to make labels in the data frame into a list (i.e. 0 8 => [0, 8])\ndef labels_to_list(labels): return list(map(int, labels.split()))\n\n# Process train & test set into an array format\nX_train = np.array([x for x in X_df_train['fv']])\nY_train = np.array([list (X_df_train['category_id'].values)]).reshape(X_train.shape[0],1)\n# Y_train=Y_train.reshape(2002,1)\n# Y_train = np.array([labels_to_list(y) for y in X_df_train['category_id']])\n\n\n# Check shape of array-format train & test set\nprint(\"X_train: \", X_train.shape)\nprint(\"Y_train: \", Y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"id":"MBL_RQO7o1ot","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Load packages for splitting train & validation set, XGBoost classifier, 1-of-K encoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom xgboost import XGBClassifier\n\n# Package for estimating a time taken\nimport time; t=time.time()","execution_count":null,"outputs":[]},{"metadata":{"id":"PCHCX2Wro1ov","colab_type":"code","outputId":"5a27b1a3-01a1-4477-d5ff-53a27e5d2465","colab":{},"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\n\n# Convert list of labels to follow 1-of-K coding scheme\none_of_K_encoder = MultiLabelBinarizer()\nY_train_ = one_of_K_encoder.fit_transform(Y_train)\n\n# Split train set into 8:2 (train : validation)\nrandom_state = np.random.RandomState(999)\nX_train_, X_test_, Y_train_, Y_test_ = train_test_split(X_train, Y_train_, test_size=0.2, \n                                                        #stratify=['category_id'],\n                                                        random_state=random_state)\n\nmx_depth=9\n# Train the XGBoost classifier\nclassifier = OneVsRestClassifier(XGBClassifier(num_class=len(classes), \n                                               gamma=0.024, \n                                               learning_rate=0.3, \n                                               max_depth=20, \n                                               nthread=4, \n                                               n_estimators=1000,\n                                               objective=\"multi:softmax\")) # 'multi:softprob'?\n\nclassifier = OneVsRestClassifier(XGBClassifier(num_class=len(classes),\n                                base_score=0.5, colsample_bytree=0.5, \n                               gamma=0.017,learning_rate=0.15, max_delta_step=0, \n                               max_depth=mx_depth, min_child_weight=3, n_estimators=3000, \n                               nthread=-1, objective='multi:softmax', seed=0, \n                               silent=1, subsample=0.8, eta=0.3)) # class_weight='auto'\n\n# multi:softprob' target to predict probabilities (2d array)\n    \nclassifier.fit(X_train_, Y_train_)\n# Show spent time\nprint(\"Time passed: \", \"{0:.3f}\".format(time.time() - t), \"sec\")","execution_count":null,"outputs":[]},{"metadata":{"id":"edLQbnY8o1ow","colab_type":"code","outputId":"e223a60d-3774-4b68-c258-2323c537471c","colab":{},"trusted":true},"cell_type":"code","source":"print (one_of_K_encoder.classes_)","execution_count":null,"outputs":[]},{"metadata":{"id":"2kkgUF40o1oy","colab_type":"code","outputId":"97d1d110-7134-4610-8d58-675a63145b36","colab":{},"trusted":true},"cell_type":"code","source":"# Predict labels using the trained model\n# Y_predict = classifier.predict_proba(X_test_) # probs\n# Predict labels using the trained model\nY_predict = classifier.predict(X_test_) # classes \n\n\n# Show some predicted values\nprint(\"Samples of predicted labels (in 1-of-K coding scheme):\\n\", Y_predict[1:8])\nprint(\"\\nSamples of corresponding predicted labels:\\n\", one_of_K_encoder.inverse_transform(Y_predict[1:8]))","execution_count":null,"outputs":[]},{"metadata":{"id":"pQf5wy3No1oz","colab_type":"text"},"cell_type":"markdown","source":"Note that the several instances returned all zeroes, indicating that they matched none of the three labels fit"},{"metadata":{"id":"u6bpkRwso1o0","colab_type":"code","outputId":"b5099c8c-d379-4bf2-8f76-e837841e2874","colab":{},"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score # For measuring F1 score metrics\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import confusion_matrix\n\n\n# Show global F1 score & on-label F1 score\nprint(\"Overall F1 score: \", f1_score(Y_test_, Y_predict, average='micro')) \nprint(\"F1 score of each label : \", f1_score(Y_test_, Y_predict, average=None))","execution_count":null,"outputs":[]},{"metadata":{"id":"xduoLpOco1o2","colab_type":"text"},"cell_type":"markdown","source":"### CM\n- input to confusion_matrix must be a list of predictions, not OHEs (one hot encodings)."},{"metadata":{"id":"QSz49sbQo1o2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import confusion_matrix\n\n# labels = np.array([0, 1,2])\nlabels= one_of_K_encoder.classes_\n\ndef cm_analysis(y_true, y_pred, labels, ymap=None, figsize=(10,10)):\n    \"\"\"\n    Generate matrix plot of confusion matrix with pretty annotations.\n    The plot image is saved to disk.\n    args: \n      y_true:    true label of the data, with shape (nsamples,)\n      y_pred:    prediction of the data, with shape (nsamples,)\n      filename:  filename of figure file to save\n      labels:    string array, name the order of class labels in the confusion matrix.\n                 use `clf.classes_` if using scikit-learn models.\n                 with shape (nclass,).\n      ymap:      dict: any -> string, length == nclass.\n                 if not None, map the labels & ys to more understandable strings.\n                 Caution: original y_true, y_pred and labels must align.\n      figsize:   the size of the figure plotted.\n    \"\"\"\n    if ymap is not None:\n        y_pred = [ymap[yi] for yi in y_pred]\n        y_true = [ymap[yi] for yi in y_true]\n        labels = [ymap[yi] for yi in labels]\n    cm = confusion_matrix(y_true, y_pred, labels=None)\n    cm_sum = np.sum(cm, axis=1, keepdims=True)\n    cm_perc = cm / cm_sum.astype(float) * 100\n    annot = np.empty_like(cm).astype(str)\n    nrows, ncols = cm.shape\n    for i in range(nrows):\n        for j in range(ncols):\n            c = cm[i, j]\n            p = cm_perc[i, j]\n            if i == j:\n                s = cm_sum[i]\n                annot[i, j] = '%.1f%%\\n%d/%d' % (p, c, s)\n            elif c == 0:\n                annot[i, j] = ''\n            else:\n                annot[i, j] = '%.1f%%\\n%d' % (p, c)\n    cm = pd.DataFrame(cm, index=labels, columns=labels)\n    cm.index.name = 'Actual'\n    cm.columns.name = 'Predicted'\n    fig, ax = plt.subplots(figsize=figsize)\n    sns.heatmap(cm, annot=annot, fmt='', ax=ax)\n#     plt.savefig(filename)","execution_count":null,"outputs":[]},{"metadata":{"id":"XO-4W5J-o1o3","colab_type":"code","outputId":"7b2da6d0-8cfc-4755-d906-3615ff4c3580","colab":{},"trusted":true},"cell_type":"code","source":"cm_analysis(Y_test_.argmax(axis=1), Y_predict.argmax(axis=1),labels)","execution_count":null,"outputs":[]},{"metadata":{"id":"jxeRF26Vo1o4","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"GzyF9G1So1o6","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"UCQ3qKjTo1o7","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"q8ZltP0Ho1o7","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"wXgNgP5Go1o8","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"8YllTORto1o8","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"HrMxy65Do1o9","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"0pG2V5JNo1o9","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"bmqqkWpXo1o-","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"Qpv6TyZVo1pA","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"KFviki0zo1pA","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"DComiADco1pB","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"ISHT5PS1o1pB","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"1mQmD6bco1pB","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"er0c679Oo1pC","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"Z5Bh65X4o1pD","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"pe8p7EXio1pD","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"ZV3aLlVro1pE","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"8sE9K9qRo1pE","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"QIdOqeVNo1pG","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"UrtGBdmUo1pG","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]}],"metadata":{"colab":{"name":"Copy of vgg-lightgbm-solution.ipynb","version":"0.3.2","provenance":[],"include_colab_link":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}