{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 5-fold EfficientNetB5 Regression with TTAx10\n## Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport cv2\nimport shutil\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport multiprocessing as mp\nimport matplotlib.pyplot as plt\nfrom tensorflow import set_random_seed\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom keras.models import Model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense, GlobalAveragePooling2D, Input\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    set_random_seed(0)\n\nseed = 0\nseed_everything(seed)\n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\nfrom efficientnet import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-07T02:24:14.995584Z","iopub.execute_input":"2023-11-07T02:24:14.995949Z","iopub.status.idle":"2023-11-07T02:24:15.013796Z","shell.execute_reply.started":"2023-11-07T02:24:14.99589Z","shell.execute_reply":"2023-11-07T02:24:15.012704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"_kg_hide-output":true}},{"cell_type":"code","source":"pathDP = '/kaggle/input/datapredict/1000 รูป 2 NidexAFC210/'\nfile_names_DP = os.listdir(path)\n\n# สร้าง DataFrame จากรายชื่อไฟล์\ndfDP = pd.DataFrame({'File Names': file_names_DP})","metadata":{"execution":{"iopub.status.busy":"2023-11-07T02:34:59.062247Z","iopub.execute_input":"2023-11-07T02:34:59.062553Z","iopub.status.idle":"2023-11-07T02:34:59.07023Z","shell.execute_reply.started":"2023-11-07T02:34:59.06251Z","shell.execute_reply":"2023-11-07T02:34:59.069299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hold_out_set = pd.read_csv('../input/aptos-data-split/hold-out.csv')\nX_train = hold_out_set[hold_out_set['set'] == 'train']\nX_val = hold_out_set[hold_out_set['set'] == 'validation']\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint('Number of train samples: ', X_train.shape[0])\nprint('Number of validation samples: ', X_val.shape[0])\nprint('Number of test samples: ', test.shape[0])\nprint('Number of DataPredict:', dfDP.shape[0])\n\n# Preprocecss data\nX_train[\"id_code\"] = X_train[\"id_code\"].apply(lambda x: x + \".png\")\nX_val[\"id_code\"] = X_val[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\ndisplay(dfDP.head())","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-07T02:35:00.508506Z","iopub.execute_input":"2023-11-07T02:35:00.508847Z","iopub.status.idle":"2023-11-07T02:35:00.542245Z","shell.execute_reply.started":"2023-11-07T02:35:00.508789Z","shell.execute_reply":"2023-11-07T02:35:00.541398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{}},{"cell_type":"code","source":"# Model parameters\nHEIGHT = 224\nWIDTH = 224\nCHANNELS = 3\nTTA_STEPS = 10\n\nweights_path_list = ['../input/aptos-5fold-224-oldnew/effNetB5_img224_fold1.h5', \n                     '../input/aptos-5fold-224-oldnew/effNetB5_img224_fold2.h5',\n                     '../input/aptos-5fold-224-oldnew/effNetB5_img224_fold3.h5', \n                     '../input/aptos-5fold-224-oldnew/effNetB5_img224_fold4.h5',\n                     '../input/aptos-5fold-224-oldnew/effNetB5_img224_fold5.h5']","metadata":{"execution":{"iopub.status.busy":"2023-11-07T02:35:05.955511Z","iopub.execute_input":"2023-11-07T02:35:05.95584Z","iopub.status.idle":"2023-11-07T02:35:05.960731Z","shell.execute_reply.started":"2023-11-07T02:35:05.955794Z","shell.execute_reply":"2023-11-07T02:35:05.959676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ndef plot_confusion_matrix(train, validation, labels=labels):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 7))\n    train_cnf_matrix = confusion_matrix(train_labels, train_preds)\n    validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)\n\n    train_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\n    validation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\n    train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\n    validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\n    sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\",ax=ax1).set_title('Train')\n    sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2).set_title('Validation')\n    plt.show()\n    \ndef evaluate_model(train, validation):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    print(\"Train        Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds, train_labels, weights='quadratic'))\n    print(\"Validation   Cohen Kappa score: %.3f\" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))\n    print(\"Complete set Cohen Kappa score: %.3f\" % cohen_kappa_score(np.append(train_preds, validation_preds), np.append(train_labels, validation_labels), weights='quadratic'))\n\ndef classify(x):\n    if x < 0.5:\n        return 0\n    elif x < 1.5:\n        return 1\n    elif x < 2.5:\n        return 2\n    elif x < 3.5:\n        return 3\n    return 4\n\ndef ensemble_preds(model_list, generator):\n    preds_ensemble = []\n    for model in model_list:\n        generator.reset()\n        preds = model.predict_generator(generator, steps=generator.n)\n        preds_ensemble.append(preds)\n\n    return np.mean(preds_ensemble, axis=0)\n\ndef apply_tta(model, generator, steps=5):\n    step_size = generator.n//generator.batch_size\n    preds_tta = []\n    for i in range(steps):\n        generator.reset()\n        preds = model.predict_generator(generator, steps=step_size)\n        preds_tta.append(preds)\n\n    return np.mean(preds_tta, axis=0)\n\ndef test_ensemble_preds(model_list, generator, steps=5):\n    preds_ensemble = []\n    for model in model_list:\n        preds = apply_tta(model, generator, steps)\n        preds_ensemble.append(preds)\n\n    return np.mean(preds_ensemble, axis=0)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-07T02:35:07.256384Z","iopub.execute_input":"2023-11-07T02:35:07.256717Z","iopub.status.idle":"2023-11-07T02:35:07.278571Z","shell.execute_reply.started":"2023-11-07T02:35:07.256657Z","shell.execute_reply":"2023-11-07T02:35:07.277776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-procecess images","metadata":{}},{"cell_type":"code","source":"new_data_base_path = '../input/aptos2019-blindness-detection/train_images/'\ntest_base_path = '../input/aptos2019-blindness-detection/test_images/'\ntrain_dest_path = 'base_dir/train_images/'\nvalidation_dest_path = 'base_dir/validation_images/'\n# test_dest_path =  'base_dir/test_images/'\ntest_dest_path =  pathDP\n\n# Making sure directories don't exist\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)\n    \n# Creating train, validation and test directories\nos.makedirs(train_dest_path)\nos.makedirs(validation_dest_path)\nos.makedirs(test_dest_path)\n\ndef crop_image(img, tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n            \n        return img\n\ndef circle_crop(img):\n    img = crop_image(img)\n\n    height, width, depth = img.shape\n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n\n    x = width//2\n    y = height//2\n    r = np.amin((x, y))\n\n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x, y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image(img)\n\n    return img\n        \ndef preprocess_image(image_id, base_path, save_path, HEIGHT=HEIGHT, WIDTH=WIDTH, sigmaX=10):\n    image = cv2.imread(base_path + image_id)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = circle_crop(image)\n    image = cv2.resize(image, (HEIGHT, WIDTH))\n    cv2.imwrite(save_path + image_id, image)\n        \ndef preprocess_data(df, HEIGHT=HEIGHT, WIDTH=WIDTH, sigmaX=10):\n    df = df.reset_index()\n    for i in range(df.shape[0]):\n        item = df.iloc[i]\n        image_id = item['id_code']\n        item_set = item['set']\n        if item_set == 'train':\n            preprocess_image(image_id, new_data_base_path, train_dest_path)\n        if item_set == 'validation':\n            preprocess_image(image_id, new_data_base_path, validation_dest_path)\n        \ndef preprocess_test(df, base_path=test_base_path, save_path=test_dest_path, HEIGHT=HEIGHT, WIDTH=WIDTH, sigmaX=10):\n    df = df.reset_index()\n    for i in range(df.shape[0]):\n        image_id = df.iloc[i]['id_code']\n        preprocess_image(image_id, base_path, save_path)\n\nn_cpu = mp.cpu_count()\ntrain_n_cnt = X_train.shape[0] // n_cpu\nval_n_cnt = X_val.shape[0] // n_cpu\ntest_n_cnt = test.shape[0] // n_cpu\n\n# Pre-procecss old data train set\npool = mp.Pool(n_cpu)\ndfs = [X_train.iloc[train_n_cnt*i:train_n_cnt*(i+1)] for i in range(n_cpu)]\ndfs[-1] = X_train.iloc[train_n_cnt*(n_cpu-1):]\nres = pool.map(preprocess_data, [x_df for x_df in dfs])\npool.close()\n\n# Pre-procecss validation set\npool = mp.Pool(n_cpu)\ndfs = [X_val.iloc[val_n_cnt*i:val_n_cnt*(i+1)] for i in range(n_cpu)]\ndfs[-1] = X_val.iloc[val_n_cnt*(n_cpu-1):] \nres = pool.map(preprocess_data, [x_df for x_df in dfs])\npool.close()\n\n# Pre-procecss test set\npool = mp.Pool(n_cpu)\ndfs = [test.iloc[test_n_cnt*i:test_n_cnt*(i+1)] for i in range(n_cpu)]\ndfs[-1] = test.iloc[test_n_cnt*(n_cpu-1):] \nres = pool.map(preprocess_test, [x_df for x_df in dfs])\npool.close()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-07T02:35:11.748917Z","iopub.execute_input":"2023-11-07T02:35:11.74922Z","iopub.status.idle":"2023-11-07T02:35:11.811211Z","shell.execute_reply.started":"2023-11-07T02:35:11.749177Z","shell.execute_reply":"2023-11-07T02:35:11.810092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data generator","metadata":{}},{"cell_type":"code","source":"datagen=ImageDataGenerator(rescale=1./255, \n                           rotation_range=360,\n                           horizontal_flip=True,\n                           vertical_flip=True)\n\ntrain_generator=datagen.flow_from_dataframe(\n                        dataframe=X_train,\n                        directory=train_dest_path,\n                        x_col=\"id_code\",\n                        y_col=\"diagnosis\",\n                        class_mode=\"raw\",\n                        batch_size=1,\n                        shuffle=False,\n                        target_size=(HEIGHT, WIDTH),\n                        seed=seed)\n\nvalid_generator=datagen.flow_from_dataframe(\n                        dataframe=X_val,\n                        directory=validation_dest_path,\n                        x_col=\"id_code\",\n                        y_col=\"diagnosis\",\n                        class_mode=\"raw\",\n                        batch_size=1,\n                        shuffle=False,\n                        target_size=(HEIGHT, WIDTH),\n                        seed=seed)\n\ntest_generator=datagen.flow_from_dataframe(  \n                       dataframe=test,\n                       directory=test_dest_path,\n                       x_col=\"id_code\",\n                       batch_size=1,\n                       class_mode=None,\n                       shuffle=False,\n                       target_size=(HEIGHT, WIDTH),\n                       seed=seed)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T22:25:31.937377Z","iopub.execute_input":"2023-06-29T22:25:31.937725Z","iopub.status.idle":"2023-06-29T22:25:32.039671Z","shell.execute_reply.started":"2023-06-29T22:25:31.937666Z","shell.execute_reply":"2023-06-29T22:25:32.03885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model(input_shape, weights_path):\n    input_tensor = Input(shape=input_shape)\n    base_model = EfficientNetB5(weights=None, \n                                include_top=False,\n                                input_tensor=input_tensor)\n\n    x = GlobalAveragePooling2D()(base_model.output)\n    final_output = Dense(1, activation='linear', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    model.load_weights(weights_path)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-29T22:25:32.041054Z","iopub.execute_input":"2023-06-29T22:25:32.041563Z","iopub.status.idle":"2023-06-29T22:25:32.049973Z","shell.execute_reply.started":"2023-06-29T22:25:32.041502Z","shell.execute_reply":"2023-06-29T22:25:32.049163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = []\n\nfor weights_path in weights_path_list:\n    model_list.append(create_model(input_shape=(HEIGHT, WIDTH, CHANNELS), weights_path=weights_path))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T22:25:32.051813Z","iopub.execute_input":"2023-06-29T22:25:32.052676Z","iopub.status.idle":"2023-06-29T22:28:21.907396Z","shell.execute_reply.started":"2023-06-29T22:25:32.052623Z","shell.execute_reply":"2023-06-29T22:28:21.906535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train predictions\npreds_ensemble = ensemble_preds(model_list, train_generator)\npreds_ensemble = [classify(x) for x in preds_ensemble]\ntrain_preds = pd.DataFrame({'label':train_generator.labels, 'pred':preds_ensemble})\n\n# Validation predictions\npreds_ensemble = ensemble_preds(model_list, valid_generator)\npreds_ensemble = [classify(x) for x in preds_ensemble]\nvalidation_preds = pd.DataFrame({'label':valid_generator.labels, 'pred':preds_ensemble})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T22:28:21.909435Z","iopub.execute_input":"2023-06-29T22:28:21.909793Z","iopub.status.idle":"2023-06-29T22:41:21.302656Z","shell.execute_reply.started":"2023-06-29T22:28:21.909742Z","shell.execute_reply":"2023-06-29T22:41:21.30134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation","metadata":{}},{"cell_type":"markdown","source":"## Confusion Matrix\n\n### Original thresholds","metadata":{}},{"cell_type":"code","source":"plot_confusion_matrix((train_preds['label'], train_preds['pred']), (validation_preds['label'], validation_preds['pred']))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T22:41:21.304816Z","iopub.execute_input":"2023-06-29T22:41:21.305156Z","iopub.status.idle":"2023-06-29T22:41:22.356383Z","shell.execute_reply.started":"2023-06-29T22:41:21.305099Z","shell.execute_reply":"2023-06-29T22:41:22.355553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Quadratic Weighted Kappa","metadata":{}},{"cell_type":"code","source":"evaluate_model((train_preds['label'], train_preds['pred']), (validation_preds['label'], validation_preds['pred']))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T22:41:22.358351Z","iopub.execute_input":"2023-06-29T22:41:22.359309Z","iopub.status.idle":"2023-06-29T22:41:22.400994Z","shell.execute_reply.started":"2023-06-29T22:41:22.359246Z","shell.execute_reply":"2023-06-29T22:41:22.400136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply model to test set and output predictions","metadata":{}},{"cell_type":"code","source":"preds = test_ensemble_preds(model_list, test_generator, TTA_STEPS)\npredictions = [classify(x) for x in preds]\n\nresults = pd.DataFrame({'id_code':test['id_code'], 'diagnosis':predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T22:41:22.402864Z","iopub.execute_input":"2023-06-29T22:41:22.403697Z","iopub.status.idle":"2023-06-29T23:46:31.746277Z","shell.execute_reply.started":"2023-06-29T22:41:22.403641Z","shell.execute_reply":"2023-06-29T23:46:31.745322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning created directories\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-06-29T23:46:31.75466Z","iopub.execute_input":"2023-06-29T23:46:31.757597Z","iopub.status.idle":"2023-06-29T23:46:32.135803Z","shell.execute_reply.started":"2023-06-29T23:46:31.757537Z","shell.execute_reply":"2023-06-29T23:46:32.129566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions class distribution","metadata":{}},{"cell_type":"code","source":"fig = plt.subplots(sharex='col', figsize=(24, 8.7))\nsns.countplot(x=\"diagnosis\", data=results, palette=\"GnBu_d\").set_title('Test')\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T23:46:32.137918Z","iopub.execute_input":"2023-06-29T23:46:32.138277Z","iopub.status.idle":"2023-06-29T23:46:32.751292Z","shell.execute_reply.started":"2023-06-29T23:46:32.138224Z","shell.execute_reply":"2023-06-29T23:46:32.750428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv('submission.csv', index=False)\ndisplay(results.head())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-29T23:46:32.753221Z","iopub.execute_input":"2023-06-29T23:46:32.754055Z","iopub.status.idle":"2023-06-29T23:46:32.929516Z","shell.execute_reply.started":"2023-06-29T23:46:32.753994Z","shell.execute_reply":"2023-06-29T23:46:32.928505Z"},"trusted":true},"execution_count":null,"outputs":[]}]}