{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\nINPUT_PATH = '/kaggle/input'\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-16T06:09:18.274108Z","iopub.execute_input":"2021-08-16T06:09:18.274617Z","iopub.status.idle":"2021-08-16T06:09:18.285930Z","shell.execute_reply.started":"2021-08-16T06:09:18.274491Z","shell.execute_reply":"2021-08-16T06:09:18.284704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.datasets import mnist\nimport tensorflow_hub as hub\n\nfrom tensorflow.keras.applications.mobilenet import MobileNet, preprocess_input\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation, Conv2D, GlobalAveragePooling2D\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, plot_roc_curve, roc_curve, auc, roc_auc_score\nfrom scipy import interp\nfrom itertools import cycle\nimport matplotlib.pyplot as plt\nimport timeit\n\n%matplotlib inline\n\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:18.287495Z","iopub.execute_input":"2021-08-16T06:09:18.287945Z","iopub.status.idle":"2021-08-16T06:09:20.136154Z","shell.execute_reply.started":"2021-08-16T06:09:18.287909Z","shell.execute_reply":"2021-08-16T06:09:20.135309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\navail_gpu = True\nif \"GPU\" not in device_name:\n    avail_gpu = False\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))\n\ninput_data_path = os.path.join(INPUT_PATH, 'diabetic-retinopathy-resized')\ntest_data = os.path.join(input_data_path, 'resized_train', 'resized_train')\ntest_cropped_data = os.path.join(input_data_path, 'resized_train_cropped','resized_train_cropped')\nprint(test_cropped_data)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:20.138233Z","iopub.execute_input":"2021-08-16T06:09:20.138582Z","iopub.status.idle":"2021-08-16T06:09:20.782565Z","shell.execute_reply.started":"2021-08-16T06:09:20.138543Z","shell.execute_reply":"2021-08-16T06:09:20.781602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load pretrained model - deprecated\ndef load_pretrained(weight='imagenet', include_top=False):\n    mobileNetModel = MobileNet(weights=weight, include_top=include_top)\n    model = Sequential()\n    model.add(mobileNetModel)\n    if not include_top:\n        model.add(GlobalAveragePooling2D())\n        model.add(Dense(5, activation='softmax'))\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:20.784468Z","iopub.execute_input":"2021-08-16T06:09:20.784861Z","iopub.status.idle":"2021-08-16T06:09:20.795336Z","shell.execute_reply.started":"2021-08-16T06:09:20.784805Z","shell.execute_reply":"2021-08-16T06:09:20.794536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# functions\n\ndef stratified_sampling(data, n_sample=None, stratify=None, random_state=None):\n    if not n_sample or not stratify:\n        return\n    str_sampled_data = data.groupby(stratify, group_keys=False).apply(lambda x: x.sample(int(np.rint(n_sample*len(x)/len(data))), random_state=random_state)).sample(frac=1).reset_index(drop=True)\n    return str_sampled_data.reset_index(drop=True)\n\ndef data_loader(data, stratify=None, downsampling=False, upsampling=False):\n    if stratify == None:\n        train, test = train_test_split(data, test_size=0.3)\n    else:\n        train, test = train_test_split(data, test_size=0.3, stratify=data[stratify])\n    if downsampling:\n        min_cnt = min(train[stratify].value_counts())\n        train = train.groupby(stratify, group_keys=False).apply(lambda x: x.sample(min_cnt)).sample(frac=1).reset_index(drop=True)\n    if upsampling:\n        max_cnt = max(train[stratify].value_counts())\n        train = train.groupby(stratify, group_keys=False).apply(lambda x: x.sample(max_cnt, replace=True)).sample(frac=1).reset_index(drop=True)\n    train[['level']].hist(figsize=(15, 15))\n    return train, test\n    \ndef micro_f1(y_true, y_pred):\n    y_true = np.array([np.argmax(p) for p in y_true]).reshape(-1, 1)\n    y_pred = np.array([np.argmax(p) for p in y_pred]).reshape(-1, 1)\n    return f1_score(y_true, y_pred, average='micro')\n\ndef get_steps(num_samples, batch_size):\n    if (num_samples % batch_size) > 0:\n        return (num_samples // batch_size) + 1\n    else:\n        return num_samples // batch_size\n\ndef img2vec(path, names, image_size = None):\n    vecs = []\n    for name in names:\n        img = cv2.cvtColor(cv2.imread(os.path.join(path, name+'.jpeg')), cv2.COLOR_BGR2RGB)\n        if image_size:\n            img = cv2.resize(img, image_size, interpolation = cv2.INTER_CUBIC)\n        vecs.append(img)\n    return np.array(vecs)\n\ndef stratified_sampling(data, n_sample=None, stratify=None, random_state=None):\n    if not n_sample or not stratify:\n        return\n    tmp_data = data.groupby(stratify, group_keys=False).apply(lambda x: x.sample(int(np.rint(n_sample*len(x)/len(data))), random_state=random_state)).sample(frac=1).reset_index(drop=True)\n    return tmp_data\n\ndef process(df, path = '../input/diabetic-retinopathy-resized/resized_train_cropped/resized_train_cropped', image_size=None):\n    feature_vec = img2vec(path, df['image'].values, image_size=image_size)\n    one_hot = pd.get_dummies(df['level'])\n    return feature_vec, one_hot\n\ndef model_train(model_loader, x_train, x_label, y_test, y_label, optimizer=None, loss=None, epochs=None, n_layer=0):\n    logs= []\n    for opt in optimizer:\n        for l in loss:\n            for e in epochs:\n                for i in range(n_layer):\n                    model = Sequential()\n                    model.add(model_loader(include_top=False))\n                    model.add(GlobalAveragePooling2D())\n                    model.add(Dense(5, activation='softmax'))\n                    for j in range(i+1):\n                        model.layers[j].trainable = True\n                    print(f'optimizer: {opt}, loss_fn:{l}, epochs: {e}, train_layer: {j}')\n                    model.compile(optimizer=opt, loss=l, metrics=['acc'])\n                    model.fit(x_train, x_label, epochs=e)\n                    prediction = model.predict(y_test)\n                    print(f'f1_score: {micro_f1(y_label, prediction)}')\n                    plot_multi_roc_curve(y_label, prediction)\n                    del model\n                    gc.collect()\n                    log = f'optimizer: {opt}, loss_fn:{l}, epochs: {e}, train_layer: {j}, micro_f1_score: {micro_f1(y_label, prediction)}'\n                    logs.append(log)\n    return logs\n\ndef plot_multi_roc_curve(y_true, y_pred):\n    n_classes = y_true.shape[1]\n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    for i in range(5):\n        fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_pred[:, i])\n        roc_auc[i] = auc(fpr[i], tpr[i])\n    # Compute micro-average ROC curve and ROC area\n    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_true.ravel(), y_pred.ravel())\n    roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n    # First aggregate all false positive rates\n    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n\n    # Then interpolate all ROC curves at this points\n    mean_tpr = np.zeros_like(all_fpr)\n    for i in range(n_classes):\n        mean_tpr += interp(all_fpr, fpr[i], tpr[i])\n\n    # Finally average it and compute AUC\n    mean_tpr /= n_classes\n\n    fpr[\"macro\"] = all_fpr\n    tpr[\"macro\"] = mean_tpr\n    roc_auc[\"macro\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n    lw = 5\n    # Plot all ROC curves\n    plt.figure()\n    plt.plot(fpr[\"micro\"], tpr[\"micro\"],\n             label='micro-average ROC curve (area = {0:0.2f})'\n                   ''.format(roc_auc[\"micro\"]),\n             color='deeppink', linestyle=':', linewidth=4)\n\n    plt.plot(fpr[\"macro\"], tpr[\"macro\"],\n             label='macro-average ROC curve (area = {0:0.2f})'\n                   ''.format(roc_auc[\"macro\"]),\n             color='navy', linestyle=':', linewidth=4)\n\n    colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])\n    for i, color in zip(range(n_classes), colors):\n        plt.plot(fpr[i], tpr[i], color=color, lw=lw,\n                 label='ROC curve of class {0} (area = {1:0.2f})'\n                 ''.format(i, roc_auc[i]))\n\n    plt.plot([0, 1], [0, 1], 'k--', lw=lw)\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Some extension of Receiver operating characteristic to multi-class')\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:20.797413Z","iopub.execute_input":"2021-08-16T06:09:20.797781Z","iopub.status.idle":"2021-08-16T06:09:20.829829Z","shell.execute_reply.started":"2021-08-16T06:09:20.797745Z","shell.execute_reply":"2021-08-16T06:09:20.828862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset 준비\nlabels = pd.read_csv(os.path.join(input_data_path, 'trainLabels_cropped.csv'))\nlabels = labels.drop(columns=['Unnamed: 0', 'Unnamed: 0.1'])","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:20.831085Z","iopub.execute_input":"2021-08-16T06:09:20.831478Z","iopub.status.idle":"2021-08-16T06:09:20.876272Z","shell.execute_reply.started":"2021-08-16T06:09:20.831444Z","shell.execute_reply":"2021-08-16T06:09:20.875539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MobileNet Model","metadata":{}},{"cell_type":"markdown","source":"## Downsampling","metadata":{}},{"cell_type":"code","source":"random_df = stratified_sampling(labels, n_sample=10000, stratify='level', random_state=312)\ntrain, test = data_loader(random_df, stratify='level', downsampling=True)\nx_train, x_label = process(train, path=test_cropped_data, image_size = (224, 224))\ny_test, y_label = process(test, path=test_cropped_data, image_size = (224, 224))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:09:20.877415Z","iopub.execute_input":"2021-08-16T06:09:20.877800Z","iopub.status.idle":"2021-08-16T06:10:26.532587Z","shell.execute_reply.started":"2021-08-16T06:09:20.877764Z","shell.execute_reply":"2021-08-16T06:10:26.531762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_pretrained(include_top=True)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:26.535265Z","iopub.execute_input":"2021-08-16T06:10:26.535597Z","iopub.status.idle":"2021-08-16T06:10:27.710050Z","shell.execute_reply.started":"2021-08-16T06:10:26.535569Z","shell.execute_reply":"2021-08-16T06:10:27.709226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:27.711964Z","iopub.execute_input":"2021-08-16T06:10:27.712326Z","iopub.status.idle":"2021-08-16T06:10:27.880242Z","shell.execute_reply.started":"2021-08-16T06:10:27.712288Z","shell.execute_reply":"2021-08-16T06:10:27.879464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MobileNet은 총 1,000개의 class를 가지고 있는 classifier가지고 있기 때문에 \n## 기존의 classifier를 때어주는 작업이 필요하다.```include_top=False```","metadata":{}},{"cell_type":"code","source":"model = load_pretrained()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:27.881440Z","iopub.execute_input":"2021-08-16T06:10:27.881793Z","iopub.status.idle":"2021-08-16T06:10:28.587379Z","shell.execute_reply.started":"2021-08-16T06:10:27.881757Z","shell.execute_reply":"2021-08-16T06:10:28.586609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 기존 weight를 그대로 사용","metadata":{}},{"cell_type":"code","source":"no_train_prediction = model.predict(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:28.588604Z","iopub.execute_input":"2021-08-16T06:10:28.588946Z","iopub.status.idle":"2021-08-16T06:10:32.460984Z","shell.execute_reply.started":"2021-08-16T06:10:28.588909Z","shell.execute_reply":"2021-08-16T06:10:32.459740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"no_train_prediction","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:32.462991Z","iopub.execute_input":"2021-08-16T06:10:32.463602Z","iopub.status.idle":"2021-08-16T06:10:32.474066Z","shell.execute_reply.started":"2021-08-16T06:10:32.463561Z","shell.execute_reply":"2021-08-16T06:10:32.473233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_label.values","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:32.477290Z","iopub.execute_input":"2021-08-16T06:10:32.477904Z","iopub.status.idle":"2021-08-16T06:10:32.489278Z","shell.execute_reply.started":"2021-08-16T06:10:32.477868Z","shell.execute_reply":"2021-08-16T06:10:32.488444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"micro_f1(y_label.values, no_train_prediction)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:32.490841Z","iopub.execute_input":"2021-08-16T06:10:32.491448Z","iopub.status.idle":"2021-08-16T06:10:32.537384Z","shell.execute_reply.started":"2021-08-16T06:10:32.491408Z","shell.execute_reply":"2021-08-16T06:10:32.536599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Layer Trainable","metadata":{}},{"cell_type":"code","source":"optimizers = ['adam']\nloss_fn = ['categorical_crossentropy']\nepochs = [5, 30]\nwith tf.device('gpu:0'):\n    logs = model_train(MobileNet, x_train, x_label.values, y_test, y_label.values, optimizer=optimizers, loss=loss_fn, epochs=epochs, n_layer=3)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:10:32.545787Z","iopub.execute_input":"2021-08-16T06:10:32.547749Z","iopub.status.idle":"2021-08-16T06:16:36.526791Z","shell.execute_reply.started":"2021-08-16T06:10:32.547700Z","shell.execute_reply":"2021-08-16T06:16:36.525924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for log in logs:\n    print(log)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:16:36.528093Z","iopub.execute_input":"2021-08-16T06:16:36.528445Z","iopub.status.idle":"2021-08-16T06:16:36.534621Z","shell.execute_reply.started":"2021-08-16T06:16:36.528410Z","shell.execute_reply":"2021-08-16T06:16:36.533492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upsampling","metadata":{}},{"cell_type":"code","source":"optimizers = ['adam']\nloss_fn = ['categorical_crossentropy']\nepochs = [10, 30]\nrandom_df = stratified_sampling(labels, n_sample=2000, stratify='level', random_state=312)\ntrain, test = data_loader(random_df, stratify='level', upsampling=True)\nx_train, x_label = process(train, path=test_cropped_data, image_size = (224, 224))\ny_test, y_label = process(test, path=test_cropped_data, image_size = (224, 224))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:18:03.176986Z","iopub.execute_input":"2021-08-16T06:18:03.177331Z","iopub.status.idle":"2021-08-16T06:19:21.698433Z","shell.execute_reply.started":"2021-08-16T06:18:03.177300Z","shell.execute_reply":"2021-08-16T06:19:21.697598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('gpu:0'):\n    logs = model_train(MobileNet, x_train, x_label.values, y_test, y_label.values, optimizer=optimizers, loss=loss_fn, epochs=epochs, n_layer=3)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T06:19:21.699992Z","iopub.execute_input":"2021-08-16T06:19:21.700335Z","iopub.status.idle":"2021-08-16T07:05:14.035838Z","shell.execute_reply.started":"2021-08-16T06:19:21.700297Z","shell.execute_reply":"2021-08-16T07:05:14.034985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# InceptionV3","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications.inception_v3 import InceptionV3","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:05:14.037808Z","iopub.execute_input":"2021-08-16T07:05:14.038154Z","iopub.status.idle":"2021-08-16T07:05:14.043227Z","shell.execute_reply.started":"2021-08-16T07:05:14.038118Z","shell.execute_reply":"2021-08-16T07:05:14.042434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Downsampling","metadata":{}},{"cell_type":"code","source":"random_df = stratified_sampling(labels, n_sample=10000, stratify='level', random_state=312)\ntrain, test = data_loader(random_df, stratify='level', downsampling=True)\nx_train, x_label = process(train, path=test_cropped_data, image_size = (299, 299))\ny_test, y_label = process(test, path=test_cropped_data, image_size = (299, 299))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:05:14.044782Z","iopub.execute_input":"2021-08-16T07:05:14.045139Z","iopub.status.idle":"2021-08-16T07:06:12.959417Z","shell.execute_reply.started":"2021-08-16T07:05:14.045104Z","shell.execute_reply":"2021-08-16T07:06:12.958530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('gpu:0'):\n    logs = model_train(InceptionV3, x_train, x_label.values, y_test, y_label.values, optimizer=optimizers, loss=loss_fn, epochs=epochs, n_layer=3)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:06:12.963456Z","iopub.execute_input":"2021-08-16T07:06:12.965712Z","iopub.status.idle":"2021-08-16T07:18:02.875641Z","shell.execute_reply.started":"2021-08-16T07:06:12.965664Z","shell.execute_reply":"2021-08-16T07:18:02.874781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Upsampling","metadata":{}},{"cell_type":"code","source":"random_df = stratified_sampling(labels, n_sample=2000, stratify='level', random_state=312)\ntrain, test = data_loader(random_df, stratify='level', upsampling=True)\nx_train, x_label = process(train, path=test_cropped_data, image_size = (299, 299))\ny_test, y_label = process(test, path=test_cropped_data, image_size = (299, 299))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:18:02.877112Z","iopub.execute_input":"2021-08-16T07:18:02.877456Z","iopub.status.idle":"2021-08-16T07:19:19.562488Z","shell.execute_reply.started":"2021-08-16T07:18:02.877420Z","shell.execute_reply":"2021-08-16T07:19:19.561716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('gpu:0'):\n    logs = model_train(InceptionV3, x_train, x_label.values, y_test, y_label.values, optimizer=optimizers, loss=loss_fn, epochs=epochs, n_layer=3)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T07:19:19.563847Z","iopub.execute_input":"2021-08-16T07:19:19.564180Z","iopub.status.idle":"2021-08-16T08:33:44.998979Z","shell.execute_reply.started":"2021-08-16T07:19:19.564144Z","shell.execute_reply":"2021-08-16T08:33:44.998067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(logs)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T08:33:45.001188Z","iopub.execute_input":"2021-08-16T08:33:45.001579Z","iopub.status.idle":"2021-08-16T08:33:45.005921Z","shell.execute_reply.started":"2021-08-16T08:33:45.001540Z","shell.execute_reply":"2021-08-16T08:33:45.005096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}