{"cells":[{"metadata":{},"cell_type":"markdown","source":"# APTOS 2019 Blindness Detection - EfficientNet B3 \n\n해당 competition은 모델이 안구 이미지에 따른 당뇨병성 망막증 증상의 단계를 얼마나 정확하게 식별하는가를 평가합니다. <br>\n제출되는 submission은 quadratic weight kappa를 통해 평가됩니다.\n\n다음 커널을 base로 작성하였습니다 : <br>\nhttps://www.kaggle.com/carlolepelaars/efficientnetb5-with-keras-aptos-2019/data"},{"metadata":{},"cell_type":"markdown","source":"상수 정의"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\n\nimport pandas as pd\nimport os.path\n\n# image directory path\n#../input/aptos-train-dataset/aptos-train-images/aptos-train-images/\n\n#original data : ../input/aptos2019-blindness-detection\nTRAIN_DATA_PATH = \"../input/aptos2019-blindness-detection\"\nTEST_DATA_PATH = \"../input/aptos2019-blindness-detection\"\nPREPROCESSED_IMAGE_PATH = \"./preprocessed\"\nMODEL_PATH = \"./models\"\nMODEL_FILE_NAME = \"effnet_b3_single.h5\"\n\nTRAIN_CSV_FILE_PATH = os.path.join(TRAIN_DATA_PATH, \"train.csv\")\nTRAIN_IMAGE_FILE_PATH = \"../input/aptos2019-blindness-detection/train_images\"\n\nTEST_CSV_FILE_PATH = \"../input/aptos2019-blindness-detection/test.csv\"\nTEST_IMAGE_FILE_PATH = \"../input/aptos2019-blindness-detection/test_images\"\n\n\"\"\"\n전체 커널이 제대로 돌아기는지 확인할 때 사용한다.\nsubmission이 정상적으로 진행되는지까지 학인\n\"\"\"\nCHECK_KERNEL_VALID = False\n\nIMG_WIDTH = 456\nIMG_HEIGHT = 456\nIMG_CHANNELS = 3\n\nBATCH_SIZE = 4\n\nNUM_FOLDS = 6\n\nEPOCHS = 30\nif CHECK_KERNEL_VALID:\n    EPOCHS = 1\n\nGENERATE_WEIGHTS = True # weight를 새로 생성한다.\n\nTRAIN_OVER_PRETRAINED = True # 기존의 weight에 추가 train한다.\n\nRESCALE_DN = 128.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport shutil\n\nif os.path.exists(MODEL_PATH) == False:\n    Path(MODEL_PATH).mkdir(parents=True, exist_ok=True)\n    \n\n# weight를 생성하려는 목적이 아니면 weight파일을 미리 복사해 둔다.\n\npre_models_path = \"../input/aptos-data-files\"\n\npre_model_filepath = os.path.join(pre_models_path, \"effnet_b3_single.h5\")\n\nif os.path.exists(pre_models_path):\n    for fname in os.listdir(pre_models_path):\n        filepath = os.path.join(pre_models_path, fname)        \n        if os.path.isfile(filepath):\n            if GENERATE_WEIGHTS == True and TRAIN_OVER_PRETRAINED == False:\n                if fname.find(\"h5\") > 0:\n                    continue\n            destfilepath = os.path.join(MODEL_PATH, fname)\n            print(\"Copy File \", filepath, \" >>> \", destfilepath)\n            shutil.copy(filepath, destfilepath)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV_FILE_PATH)\ndf_train['id_code'] = df_train['id_code'] + \".png\"\ndf_test = pd.read_csv(TEST_CSV_FILE_PATH)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\n#from PIL import Image, ImageDraw\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\nn = 3\n\nfix, ax = plt.subplots(n, n, figsize = (16, 16))\naxidx = 0\n\ndf_sample = df_train.sample(n * n)\nfor idx, row in df_sample.iterrows():\n    imgpath = os.path.join(TRAIN_IMAGE_FILE_PATH, row['id_code'])\n    \n    im = cv2.imread(imgpath)\n    # Note : In the case of color images, the decoded images will have the channels stored in B G R order.\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # RGB로 바꿔주지 않으면 이미지가 파랗게 나온다.\n\n    ax[int(axidx / n)][axidx % n].imshow(im)\n    axidx += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing\n\n다음 방법을 통해 이미지에서 시신경 및 문제가 될 수 있는 부분들이 큰 contrast를 가져서 뚜렷이 보이도록 처리.<br>\n\n1. 원본 이미지에서 위/아래의 검은 부분을 crop하여 제거함\n2. 경계를 강조\n3. 밝기를 높임"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\ndef crop_image_from_gray(img, tol=7):\n    \"\"\"\n    Applies masks to the orignal image and \n    returns the a preprocessed image with \n    3 channels\n    \n    (img > tolerance) 로 mask를 생성하고 np.any()를 사용해서\n    merge되는 값들 중 valid한 값이 있는 줄은 np.ix_()로 골라냄\n        -> False인 줄은 모두 제거됨\n    \"\"\"\n    # If for some reason we only have two channels\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    # If we have a normal RGB images\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\ndef preprocess_image(image, sigmaX=10):\n    \"\"\"\n    The whole preprocessing pipeline:\n    1. Read in image\n    2. Apply masks\n    3. Resize image to desired size\n    4. Add Gaussian noise to increase Robustness\n    \"\"\"\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) #RGB colorspace로 g변경\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_WIDTH, IMG_HEIGHT))\n    # cv2.addWeighted(src1, alpha, src2, beta, gamma[, dst[, dtype]]) \n    #image = cv2.addWeighted (image, 4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -4, 128)\n    \n    '''\n    cv2.addWeighted() : Calculates the weighted sum of two arrays.\n    원본 이미지에 블러처리된 원본 이미지에 음수 가중치를 준 것을 더해서 윤곽을 강조\n    (gamma로 contrast를 줌)\n    '''\n    image = cv2.addWeighted (image, 4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -4, gamma=128)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix, ax = plt.subplots(n, n, figsize = (20, 20))\n\naxidx = 0    \nfor idx, row in df_sample.iterrows():\n    filename = row['id_code']\n    imgpath = os.path.join(TRAIN_IMAGE_FILE_PATH, filename)\n    im = preprocess_image(cv2.imread(imgpath))\n    ax[int(axidx / (n))][axidx % n].imshow(im)\n    ax[int(axidx / (n))][axidx % n].set_title(row['id_code'])\n    axidx += 1\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_and_labels(model, generator):\n    \"\"\"\n    Get predictions and labels from the generator\n    \"\"\"\n    preds = []\n    labels = []\n    for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):\n        x, y = next(generator)\n        preds.append(model.predict(x))\n        labels.append(y)\n    # Flatten list of numpy arrays\n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation\n\ncompetition의 submission 평가는 QWK(Quadratic Weighted Kappa) 값으로 이루어진다.\n\n### Cohen's Kappa\n두 연구자가 얼마나 동일한 결과를 내놓는지를 수치화하는 방법이다. 두 연구자 간 일치한 결과 중에서 우연히 일치할 가능성를 제외하고, 실제로 평가가 일치한 결과가 어느 정도인지 보여주는 지표이다.<br>\nnominal(category간 거리가 같은)한 범주에 사용된다.\n\n### Cohen's weighted Kappa\nCohen's Kappa와는 다르게, ordinal(순서가 있는, 예를 들어 관절염의 5 단계(1:없음, 2:경증 ... 5:심각) 등을 표현 시) 변수를 대상으로 할 경우에는 Cohen's weighted Kappa를 사용한다. <br>\n순서(또는 단계)가 있는 변수를 판단하므로 범주(카테고리)간 거리는 서로 다르고, 두 연구자간 결과가 다를 경우에도 다름의 크기에 가중치 차이가 있을 것이다.<br>\n이런 식으로 각각 다른 비중(weight)를 두어 불일치 정도를 평가하는 것이다.\n\n각 범주간 차이에 비중을 부여하는 방법으로는 값의 차이를 그대로 사용하는 linear 방법과, 제곱해서 사용하는 quadratic 방법이 있다.<br>\n"},{"metadata":{},"cell_type":"markdown","source":"QWK가 개선되는 경우 모델을 저장하는 custom callback을 정의해서 train시 사용한다.\n\n> **sklearn.metrics.cohen_kappa_score(y1, y2, labels=None, weights=None, sample_weight=None)**\n\nCohen’s kappa: a statistic that measures inter-annotator agreement.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\nfrom keras.callbacks import Callback\n\nclass Metrics(Callback):\n    \"\"\"\n    A custom Keras callback for saving the best model\n    according to the Quadratic Weighted Kappa (QWK) metric\n    \"\"\"\n    def __init__(self, model, val_generator, model_save_filepath):\n        self.model = model\n        self.val_generator = val_generator\n        self.model_save_filepath = model_save_filepath\n        \n    def on_train_begin(self, logs={}):\n        \"\"\"\n        Initialize list of QWK scores on validation data\n        \"\"\"\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \"\"\"\n        Gets QWK score on the validation data\n        \"\"\"\n        # Get predictions and convert to integers\n        y_pred, labels = get_preds_and_labels(self.model, self.val_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        # We can use sklearns implementation of QWK straight out of the box\n        # as long as we specify weights as 'quadratic'\n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic') # QWK 방법을 사용\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save(self.model_save_filepath)\n        return","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling"},{"metadata":{},"cell_type":"markdown","source":"### callbacks\n\nGenerator를 사용해서 계속 train하므로, 더 이상 loss 개선이 되지 않을 때 멈추도록 EarlyStopping, 일정 epoch이상 개선이 안되면 learning rate를 줄여나가는 ReduceLROnPlateau, 이전에 선언한 Metrics를 사용한다."},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\ndef get_callbacks(model, val_generator, model_save_filepath):\n    # Monitor MSE to avoid overfitting and save best model\n    es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)\n    # factor : 변경 시 multiplier\n    lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,\n                           verbose=1, mode='auto', min_delta=0.0001)\n    km = Metrics(model, val_generator, model_save_filepath)\n    return [es, lr, km]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"EfficientNetB3을 base로 하고, 출력은 linear로 뽑는다.<br>\noutput이 nomial하기 때문에 linear로 출력하고 loss는 MSE로 한다."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_total_batch(num_samples, batch_size):    \n    if (num_samples % batch_size) > 0 :\n        return (num_samples // batch_size) + 1\n    else :\n        return num_samples // batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.activations import elu\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\n\nfrom efficientnet import EfficientNetB3\n\n\ndef build_model_effnet_b3(load_weights = True):\n    \"\"\"\n    A custom implementation of EfficientNetB5\n    for the APTOS 2019 competition\n    (Regression)\n    \"\"\"\n    \n    # Load in EfficientNetB5\n    effnet_b3 = EfficientNetB3(weights=None,\n                        include_top=False,\n                        input_shape=(IMG_WIDTH, IMG_HEIGHT, IMG_CHANNELS))\n    if load_weights == True:\n        effnet_b3.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b3_imagenet_1000_notop.h5')\n    \n    model = Sequential()\n    model.add(effnet_b3)\n    model.add(GlobalAveragePooling2D()) # 각각의 채널을 평균해서 변환함. 바로 FC로 변경된다.\n    model.add(Dropout(0.5))\n    model.add(Dense(5, activation=elu))\n    model.add(Dense(1, activation=\"linear\")) # 0~4까지 단계적 증감값이 출력이므로 activation은 선형으로 한다.\n    model.compile(loss='mse',\n                  optimizer=Adam(lr=0.00005), \n                  metrics=['mse', 'acc'])\n    print(model.summary())\n    return model\n\n\ndef get_model(load_weight):\n    return build_model_effnet_b3(load_weight)\n    \ndef get_pretrained_model(model_filepath):\n    #model_filepath = os.path.join(MODEL_PATH, MODEL_FILE_NAME)\n    model = get_model(load_weight = False)\n    model.load_weights(model_filepath)\n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.backend import clear_session\nimport gc\n\n# Reset Keras Session\ndef clear_memory():\n    clear_session()\n    for i in range(20):\n        gc.collect()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport psutil \n\nfrom keras import backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\n\n\ndef train_single():    \n    model_save_filepath = os.path.join(MODEL_PATH, MODEL_FILE_NAME)\n    \n    model = None\n    if TRAIN_OVER_PRETRAINED:\n        print(\"train with existing weight :\", pre_model_filepath)\n        model = get_pretrained_model(pre_model_filepath) # 이전에 훈련한 weight로 초기화된 모델을 사용한다.\n    else:\n        model = get_model() # imageNet weight로 초기화된 모델을 사용한다.\n    \n    # Add Image augmentation to our generator\n    train_datagen = ImageDataGenerator(rescale = 1./RESCALE_DN,\n                                       preprocessing_function=preprocess_image, \n                                       rotation_range=360,\n                                       horizontal_flip=True,\n                                       validation_split=0.15,\n                                       vertical_flip=True)\n    \n\n    # Use the dataframe to define train and validation generators\n    train_generator = train_datagen.flow_from_dataframe(df_train,\n                                                        x_col='id_code', \n                                                        y_col='diagnosis',\n                                                        directory = TRAIN_IMAGE_FILE_PATH,\n                                                        target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                        batch_size=BATCH_SIZE,\n                                                        class_mode='other',\n                                                        subset='training')\n\n    val_generator = train_datagen.flow_from_dataframe(df_train,\n                                                      x_col='id_code',\n                                                      y_col='diagnosis',\n                                                      directory = TRAIN_IMAGE_FILE_PATH,\n                                                      target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                      batch_size=BATCH_SIZE,\n                                                      class_mode='other',\n                                                      subset='validation')\n    \n    if TRAIN_OVER_PRETRAINED == False:\n        if GENERATE_WEIGHTS == True:\n            if os.path.exists(model_save_filepath) == True:\n                os.remove(model_save_filepath)\n\n        # skip if weight file exists and not use pre-trained\n        if os.path.exists(model_save_filepath) == True:\n            print(\">>>>>>>>>>\", model_save_filepath, \" already trained... skip!\")\n            return\n\n    train_steps = get_total_batch(train_generator.samples, BATCH_SIZE)\n    val_steps = get_total_batch(val_generator.samples, BATCH_SIZE)\n    print(\"Steps : train=\", train_steps, \" validation=\", val_steps)\n\n    # make callbacks\n    callbacks = get_callbacks(model=model, val_generator=val_generator, model_save_filepath=model_save_filepath)\n\n    # First training phase (train top layer)\n    model.fit_generator(train_generator,\n                        steps_per_epoch = train_steps,\n                        epochs = EPOCHS,\n                        validation_data = val_generator,\n                        validation_steps = val_steps,\n                        callbacks = callbacks)\n    \ndef train_models():\n    clear_memory()\n\n    train_single()\n\n    # clear used memory\n    clear_memory()\n            \n\n\ntrain_models()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{},"cell_type":"markdown","source":"#### OptimizedRounder\npredict 출력이 어느 카테고리에 포함되는지 판별하는 경계값을 scipy 최적화로 최적화 한다."},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport scipy as sp\nfrom functools import partial\n\nclass OptimizedRounder(object):\n    \"\"\"\n    An optimizer for rounding thresholds\n    to maximize Quadratic Weighted Kappa score\n    \n    regression 값으로 나온 실수값을 정수로 변경.\n    label과 비교하여  기준값(coef)이 fit 된다.\n    \"\"\"\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        \"\"\"\n        Get loss according to using current coefficients\n        \n        현재 coef로 rounding한 값을 label과의 cohen kappa score 계산한다.\n        loss함수로 사용할 것이므로 음수로 리턴해준다.\n        (값이 좋아질수록 음수가 커진다->loss가 줄어든다.)\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        \"\"\"\n        Optimize rounding thresholds\n        \"\"\"\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        # scipy를 이용 initial_coef값을 최적화한다.\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        \"\"\"\n        Make predictions with specified thresholds\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clear_memory()\n\n\"\"\"\ntrain 데이터로 최적화된 OptimizedRounder를 얻는다.\n\"\"\"\ndf_train = pd.read_csv(TRAIN_CSV_FILE_PATH)\ndf_train['id_code'] = df_train['id_code'] + \".png\"\n\nval_datagen = ImageDataGenerator(rescale = 1./RESCALE_DN,\n                                 preprocessing_function=preprocess_image)\n\nval_generator = val_datagen.flow_from_dataframe(df_train,\n                                                x_col='id_code',\n                                                y_col='diagnosis',\n                                                directory = TRAIN_IMAGE_FILE_PATH,\n                                                target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                batch_size=BATCH_SIZE,\n                                                class_mode='other')\n\n\nmodel_save_filepath = os.path.join(MODEL_PATH, MODEL_FILE_NAME)    \n\nmodel = get_pretrained_model(model_save_filepath)        \n\ny_val_preds, val_labels = get_preds_and_labels(model, val_generator)\noptR = OptimizedRounder()\noptR.fit(y_val_preds, val_labels)\ncoefficients = optR.coefficients()\n\n\nclear_memory()\n\n\ndef make_submission():\n\n    # Place holder for diagnosis column\n    test_df = pd.read_csv(TEST_CSV_FILE_PATH)\n    test_df['id_code'] = test_df['id_code'] + \".png\" # 확장자 명이 없으므로 추가해야 한다.\n    \n    test_df['diagnosis'] = np.zeros(test_df.shape[0]) \n    # For preprocessing test images\n    \n    datagen = ImageDataGenerator(rescale = 1./RESCALE_DN,\n                                 preprocessing_function=preprocess_image)\n    \n    test_generator = datagen.flow_from_dataframe(\n                                            test_df, \n                                            x_col='id_code',\n                                            y_col='diagnosis',\n                                            directory=TEST_IMAGE_FILE_PATH,\n                                            target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                            batch_size=BATCH_SIZE,\n                                            class_mode='other',\n                                            shuffle=False)\n\n    model = get_pretrained_model(model_save_filepath)\n\n    y_test, _ = get_preds_and_labels(model, test_generator)\n    \n    steps = get_total_batch(test_df.shape[0], BATCH_SIZE)\n    y_test = model.predict_generator(generator = test_generator,\n                                           steps = steps,\n                                           verbose = 0)\n    \n    y_test = optR.predict(y_test, coefficients).astype(np.uint8)\n\n    test_df['diagnosis'] = y_test\n    # Remove .png from ids\n    test_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')\n    test_df.to_csv('submission.csv', index=False)\n\nmake_submission()\nclear_memory()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}