{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n

APTOS Blindness Detection with ResNet152V2

\n\n","metadata":{"_uuid":"69f8d94a-df89-4b18-a40e-473df5142356","_cell_guid":"fa91d6c1-29dc-4bfa-a833-904c39612aa1","trusted":true}},{"cell_type":"markdown","source":"* train.csv - the training labels\n* train.zip - the training set images\n\n\n저희는 해당 train data를 training, validation, test dataset (6:2:2) 으로 다시 분류하여 사용했습니다.\n\n#### Dataset Link \n\n##### [Here](https://www.kaggle.com/c/aptos2019-blindness-detection/data)","metadata":{"_uuid":"44da2f52-5bf3-4b29-9503-bb08ce167123","_cell_guid":"e386e135-c1ea-41e6-9ebf-26a119c99cea","trusted":true}},{"cell_type":"code","source":"import os\nimport cv2\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom keras.models import Model\nfrom keras import optimizers, applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing import image_dataset_from_directory\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import InputLayer, BatchNormalization, Dropout, Flatten, Dense, Activation, MaxPool2D, Conv2D\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.utils import to_categorical\nfrom keras import optimizers\nfrom tensorflow.keras.optimizers import Adam\nfrom keras.callbacks import Callback,ModelCheckpoint,ReduceLROnPlateau\nfrom keras.models import Sequential,load_model\nfrom keras.layers import Dense, Dropout\nfrom keras.wrappers.scikit_learn import KerasClassifier\nimport keras.backend as K\n\nfrom typeguard import typechecked\nfrom typing import Optional\n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:44:02.483703Z","iopub.execute_input":"2021-06-20T07:44:02.484093Z","iopub.status.idle":"2021-06-20T07:44:02.555767Z","shell.execute_reply.started":"2021-06-20T07:44:02.484052Z","shell.execute_reply":"2021-06-20T07:44:02.551671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train= pd.read_csv('../input/aptos2019-blindness-detection/train.csv')","metadata":{"_uuid":"25174e4d-fb44-49fd-ba6c-414c45751cf7","_cell_guid":"ad02b173-835f-4dc9-a608-e542fce7c894","collapsed":false,"_kg_hide-input":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:44:10.1019Z","iopub.execute_input":"2021-06-20T07:44:10.102319Z","iopub.status.idle":"2021-06-20T07:44:10.1234Z","shell.execute_reply.started":"2021-06-20T07:44:10.102277Z","shell.execute_reply":"2021-06-20T07:44:10.122626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of train samples: ', train.shape[0]) # df.shape[0] : dataframe row count (행 개수 세기)\n\ndisplay(train.head()) # top 5 row (상위 5개 행 출력)","metadata":{"_uuid":"781a2226-f18a-450a-b3a1-b315dc447114","_cell_guid":"be810968-30ba-445d-b65a-09e11a79207d","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:44:13.752708Z","iopub.execute_input":"2021-06-20T07:44:13.75312Z","iopub.status.idle":"2021-06-20T07:44:13.773327Z","shell.execute_reply.started":"2021-06-20T07:44:13.753085Z","shell.execute_reply":"2021-06-20T07:44:13.772613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas_profiling as pp\npp.ProfileReport(train)","metadata":{"_uuid":"46496e70-bc32-4659-a8d3-3253bbd6c18f","_cell_guid":"3028840c-2ebd-4ae5-9f59-c424b076ba71","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:44:17.774421Z","iopub.execute_input":"2021-06-20T07:44:17.774854Z","iopub.status.idle":"2021-06-20T07:44:32.903294Z","shell.execute_reply.started":"2021-06-20T07:44:17.774813Z","shell.execute_reply":"2021-06-20T07:44:32.902486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(14, 5))\nax = sns.countplot(x=\"diagnosis\", data=train, palette=\"Set2\") # sns: seaborn, 카테고리별 데이터 개수 보여주는 그래프\nsns.despine() # top, right 테두리 제거\nplt.show() # 그래프 화면에 표시","metadata":{"_uuid":"e5ed4759-d367-4f10-bdb8-f0784d338209","_cell_guid":"ab7dc6d2-1296-4445-8e3d-3b527f86ffe8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:44:45.268239Z","iopub.execute_input":"2021-06-20T07:44:45.268664Z","iopub.status.idle":"2021-06-20T07:44:46.153569Z","shell.execute_reply.started":"2021-06-20T07:44:45.268627Z","shell.execute_reply":"2021-06-20T07:44:46.152672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Down sampling\n데이터의 불균형을 최소화 하기 위해,\ndata의 수가 많은 normal(0)과 moderate(2)의 데이터를 줄임","metadata":{}},{"cell_type":"code","source":"normal = train[train.diagnosis == 0].sample(n=370, random_state=1004)\nmoderate = train[train.diagnosis == 2].sample(n=370, random_state=1004)\ntotal = train[(train.diagnosis != 0) & (train.diagnosis != 2)].append(normal).append(moderate)\n\nlen(total)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:44:56.819955Z","iopub.execute_input":"2021-06-20T07:44:56.820395Z","iopub.status.idle":"2021-06-20T07:44:56.88218Z","shell.execute_reply.started":"2021-06-20T07:44:56.820351Z","shell.execute_reply":"2021-06-20T07:44:56.881397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(14, 5))\nax = sns.countplot(x=\"diagnosis\", data=total, palette=\"Set2\") # sns: seaborn, 카테고리별 데이터 개수 보여주는 그래프\nsns.despine() # top, right 테두리 제거\nplt.show() # 그래프 화면에 표시","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:44:59.369059Z","iopub.execute_input":"2021-06-20T07:44:59.36951Z","iopub.status.idle":"2021-06-20T07:45:00.587779Z","shell.execute_reply.started":"2021-06-20T07:44:59.36947Z","shell.execute_reply":"2021-06-20T07:45:00.583213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"white\") # 배경색 흰색으로 지정\ncount = 1\nplt.figure(figsize=[15, 15]) # 새로운 figure 크기 가로 세로 15인치\nfor img_name in total['id_code'][:15]: # train의 id_code열 데이터 15행까지 받아옴\n img = cv2.imread(\"../input/preprocess-nb/preprocess/1/%s.png\" % img_name)[...,[2, 1, 0]]\n plt.subplot(5, 5, count) # plt.subplot(nrow,ncol,pos) nrow : 몇 줄 ncol : 몇 칸 pos : 오른쪽, 아래 방향\n plt.imshow(img) # 그림 보여주기\n plt.title(\"Image %s\" % count) # 제목 지정\n count += 1\n \nplt.show()","metadata":{"_uuid":"9254071b-8f33-4075-a8a6-d5bd0ce88023","_cell_guid":"ba7a090c-20e2-43bd-a6ad-f66e44fe6c6b","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:57:02.54451Z","iopub.execute_input":"2021-06-20T07:57:02.544939Z","iopub.status.idle":"2021-06-20T07:57:16.054664Z","shell.execute_reply.started":"2021-06-20T07:57:02.544892Z","shell.execute_reply":"2021-06-20T07:57:16.053656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_CLASSES = total['diagnosis'].nunique() # unique value 개수 나타냄\nN_CLASSES","metadata":{"_uuid":"76ad3d22-67bb-4c35-b1fc-6ac4fe3211fc","_cell_guid":"633ced04-5d06-4d3a-8698-a4bc08939680","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:57:58.424177Z","iopub.execute_input":"2021-06-20T07:57:58.424634Z","iopub.status.idle":"2021-06-20T07:57:58.432397Z","shell.execute_reply.started":"2021-06-20T07:57:58.424574Z","shell.execute_reply":"2021-06-20T07:57:58.431484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split data\ntrain, test set으로 먼저 나눈다 (8:2)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nx = total.iloc[:,0].values # total의 첫번째 열 값 추출\ny = total.iloc[:,1].values\ny","metadata":{"_uuid":"a91218a2-eb36-4ac9-9850-e90cade1053b","_cell_guid":"b8a536cd-6c15-417f-9744-1e84c6f8852a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:00.087605Z","iopub.execute_input":"2021-06-20T07:58:00.08806Z","iopub.status.idle":"2021-06-20T07:58:00.111589Z","shell.execute_reply.started":"2021-06-20T07:58:00.088009Z","shell.execute_reply":"2021-06-20T07:58:00.106623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, train_size=0.8, random_state=80)\n# random_state로 seed 지정\n# test, train split","metadata":{"_uuid":"02161c2a-98e0-462d-9530-ea0869814ecc","_cell_guid":"2c79cd9d-ac17-4f61-8f41-429b0e457339","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:01.521736Z","iopub.execute_input":"2021-06-20T07:58:01.522184Z","iopub.status.idle":"2021-06-20T07:58:01.532478Z","shell.execute_reply.started":"2021-06-20T07:58:01.522147Z","shell.execute_reply":"2021-06-20T07:58:01.531622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.DataFrame({'id_code':x_train,'diagnosis':y_train}) # 각 열 제목과 해당 값 넣어서 dataframe 만듦\ntest = pd.DataFrame({'id_code':x_test,'diagnosis':y_test})\ntrain","metadata":{"_uuid":"42890b54-19ec-4f7a-b78c-6cda0ee8e7ad","_cell_guid":"e4ee3047-67f3-4f51-bd41-9874e8ca99b7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:02.945056Z","iopub.execute_input":"2021-06-20T07:58:02.945486Z","iopub.status.idle":"2021-06-20T07:58:03.03971Z","shell.execute_reply.started":"2021-06-20T07:58:02.945441Z","shell.execute_reply":"2021-06-20T07:58:03.038846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocecss data\ntrain[\"id_code\"] = train[\"id_code\"].apply(lambda x: x + \".png\") # train의 id_code 열 값에 .png 추가\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\ntrain['diagnosis'] = train['diagnosis'].astype('str') # train의 diagnosis 값 type을 모두 str로 변환\ntest['diagnosis'] = test['diagnosis'].astype('str')\nprint(train.head())\nprint(test.head())\nprint(len(train), len(test))","metadata":{"_uuid":"517e5ef3-700b-47a2-bf14-e8f2e7d4e3e4","_cell_guid":"b7c271ed-71d4-4c7d-9339-e05bba709e10","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:04.100124Z","iopub.execute_input":"2021-06-20T07:58:04.100555Z","iopub.status.idle":"2021-06-20T07:58:04.161147Z","shell.execute_reply.started":"2021-06-20T07:58:04.100494Z","shell.execute_reply":"2021-06-20T07:58:04.156885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model construction","metadata":{}},{"cell_type":"code","source":"train_datagen=ImageDataGenerator(rescale=1./255, # pixel 값을 0~1 사이 값으로 변환\n validation_split=0.25, # validation data 비율 지정\n horizontal_flip=True) # 상하대칭\n\ntrain_generator=train_datagen.flow_from_dataframe(\n dataframe=train,\n directory=\"../input/preprocess-nb/preprocess/1/\", # 해당 directory에서 가져옴\n x_col=\"id_code\",\n y_col=\"diagnosis\",\n batch_size=16, # 한번에 네트워크에 넘겨주는 sample의 수\n class_mode=\"categorical\",\n target_size=(224, 224),\n subset='training')","metadata":{"_uuid":"0ecc9e70-098d-48d8-9539-da74a0824b23","_cell_guid":"8afe8a2a-df67-442e-a84e-35acdd003a87","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:07.973956Z","iopub.execute_input":"2021-06-20T07:58:07.974349Z","iopub.status.idle":"2021-06-20T07:58:10.178763Z","shell.execute_reply.started":"2021-06-20T07:58:07.974313Z","shell.execute_reply":"2021-06-20T07:58:10.175618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_generator=train_datagen.flow_from_dataframe(\n dataframe=train,\n directory=\"../input/preprocess-nb/preprocess/1/\",\n x_col=\"id_code\",\n y_col=\"diagnosis\",\n batch_size=16,\n class_mode=\"categorical\", \n target_size=(224, 224),\n subset='validation')","metadata":{"_uuid":"c1bf219b-86d1-4c86-a09b-047bc47690b5","_cell_guid":"c3cb7de7-a5ea-4120-aa4e-2098c29d99ca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:11.619567Z","iopub.execute_input":"2021-06-20T07:58:11.620044Z","iopub.status.idle":"2021-06-20T07:58:11.839466Z","shell.execute_reply.started":"2021-06-20T07:58:11.619998Z","shell.execute_reply":"2021-06-20T07:58:11.83863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_datagen.flow_from_dataframe( \n dataframe=test,\n directory = \"../input/preprocess-nb/preprocess/1/\",\n x_col=\"id_code\",\n target_size=(224, 224),\n batch_size=16,\n shuffle=False,\n class_mode=None)","metadata":{"_uuid":"bf0f38f0-df8d-46b9-8738-91957d6ad621","_cell_guid":"862e98e1-edfc-45c7-b94f-092bf76e9606","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:13.240489Z","iopub.execute_input":"2021-06-20T07:58:13.24094Z","iopub.status.idle":"2021-06-20T07:58:14.428451Z","shell.execute_reply.started":"2021-06-20T07:58:13.240895Z","shell.execute_reply":"2021-06-20T07:58:14.427624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = tf.keras.applications.ResNet152V2(input_shape=(224,224,3),include_top=False,weights=\"imagenet\")","metadata":{"_uuid":"34a347da-cfc2-46a8-8b17-b55c78cf39fb","_cell_guid":"2c333bd9-5334-4327-8adc-b961c9cf8406","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:16.32564Z","iopub.execute_input":"2021-06-20T07:58:16.326048Z","iopub.status.idle":"2021-06-20T07:58:35.864625Z","shell.execute_reply.started":"2021-06-20T07:58:16.326009Z","shell.execute_reply":"2021-06-20T07:58:35.863658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freezing Layers\n\nfor layer in base_model.layers[:-10]:\n layer.trainable=False","metadata":{"_uuid":"962c3148-e65e-43ec-aa41-c46ffeba2b48","_cell_guid":"cde771a4-67f6-4fe0-a11b-1034ac44cc67","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:37.301442Z","iopub.execute_input":"2021-06-20T07:58:37.30397Z","iopub.status.idle":"2021-06-20T07:58:37.419279Z","shell.execute_reply.started":"2021-06-20T07:58:37.303922Z","shell.execute_reply":"2021-06-20T07:58:37.418376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building Model\n\nmodel=Sequential()\nmodel.add(base_model)\n# convolution, pooling을 통해 feature extraction\n# layer 152개를 쌓아서 성능 높임\n\n# 여기서부터 classifier\nmodel.add(Dropout(0.5))\n# overfitting 방지, 1-p의 확률로 특정 node를 제거하겠다.\nmodel.add(Flatten())\n# dense와 같이 분류를 위한 학습 레이어에서는 1차원 데이터 필요\n# 1차원 데이터로 바꿔주는 역\nmodel.add(BatchNormalization())\n# gradient vanishing 방지 (layer 수 많을수록 심각)\nmodel.add(Dense(256,kernel_initializer='he_uniform'))\n# clssifier, 이미지를 적당한 카테고리로 분류\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n# 신경망의 output을 결정하는 식\nmodel.add(Dropout(0.5))\nmodel.add(Dense(128,kernel_initializer='he_uniform'))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(32,kernel_initializer='he_uniform'))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Dense(5,activation='softmax')) # softmax 함수를 통해 최종 class 분류하기","metadata":{"_uuid":"064e5b8a-c97e-405e-8538-6ea49d5821ca","_cell_guid":"a0665457-04aa-4184-b0d1-77a82f836d24","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:38.49239Z","iopub.execute_input":"2021-06-20T07:58:38.492853Z","iopub.status.idle":"2021-06-20T07:58:44.7768Z","shell.execute_reply.started":"2021-06-20T07:58:38.492804Z","shell.execute_reply":"2021-06-20T07:58:44.774637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Summary\n","metadata":{}},{"cell_type":"code","source":"model.summary()","metadata":{"_uuid":"6693d11f-3aa5-4c22-95f8-9721b20bbf24","_cell_guid":"58398d70-9488-4161-a27a-3e7e26cb32fd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:44.785357Z","iopub.execute_input":"2021-06-20T07:58:44.785743Z","iopub.status.idle":"2021-06-20T07:58:44.991874Z","shell.execute_reply.started":"2021-06-20T07:58:44.785704Z","shell.execute_reply":"2021-06-20T07:58:44.9879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nfrom IPython.display import Image\nplot_model(model, to_file='convnet.png', show_shapes=True,show_layer_names=True)\nImage(filename='convnet.png')","metadata":{"_uuid":"184961f7-7c6d-4390-af31-56ff1f7c4f9e","_cell_guid":"feeb1440-30ea-4ded-94c4-a2638801f780","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:46.120564Z","iopub.execute_input":"2021-06-20T07:58:46.121008Z","iopub.status.idle":"2021-06-20T07:58:47.213602Z","shell.execute_reply.started":"2021-06-20T07:58:46.120963Z","shell.execute_reply":"2021-06-20T07:58:47.210232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# f1_score 계산식","metadata":{}},{"cell_type":"code","source":"def f1_score(y_true, y_pred): # taken from old keras source code\n true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n precision = true_positives / (predicted_positives + K.epsilon())\n recall = true_positives / (possible_positives + K.epsilon())\n f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())\n return f1_val","metadata":{"_uuid":"dd666e0a-c895-4e18-94ef-f4b3e78ae9a2","_cell_guid":"f96f30d4-2208-4a88-a5ef-1ec6822565a6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:50.276121Z","iopub.execute_input":"2021-06-20T07:58:50.276655Z","iopub.status.idle":"2021-06-20T07:58:50.293667Z","shell.execute_reply.started":"2021-06-20T07:58:50.276598Z","shell.execute_reply":"2021-06-20T07:58:50.292818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 모델 평가기준","metadata":{}},{"cell_type":"code","source":"METRICS = [\n tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n tf.keras.metrics.Precision(name='precision'),\n tf.keras.metrics.Recall(name='recall'), \n tf.keras.metrics.AUC(name='auc'),\n f1_score,\n]","metadata":{"_uuid":"84fe8f0f-76a7-4e37-ba1d-ea2e6cbee6b0","_cell_guid":"550186b3-f50e-444b-bcb6-2276118e959d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:52.35951Z","iopub.execute_input":"2021-06-20T07:58:52.359972Z","iopub.status.idle":"2021-06-20T07:58:52.500424Z","shell.execute_reply.started":"2021-06-20T07:58:52.359929Z","shell.execute_reply":"2021-06-20T07:58:52.499617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lrd = ReduceLROnPlateau(monitor = 'val_loss',patience = 2,verbose = 1,factor = 0.8, min_lr = 1e-6)\n# learning rate를 조절해주는 콜백함수\n# val_loss가 더이상 감소되지 않을 경우, 해당 함수 적용\n# patience : 최적의 monitor 값을 기준으로 몇번의 epoch를 진행하고 leraning rate를 조절할지의 값\n# factor : learning rate를 얼마나 감소시킬지 정하는 인자\n# verbose : 1 = early stopping 적용될 때, 화면에 나타냄\n# min_lr : learning rate의 하한선 지정\n# 예를 들어 patience는 3이고, 30에폭에 정확도가 99%였을 때,만약 31번째에 정확도 98%, 32번째에 98.5%, 33번째에 98%라면 모델의 개선이 (patience=3)동안 개선이 없었기에, ReduceLROnPlateau 콜백함수를 실행합니다.\n\nmcp = ModelCheckpoint('ResNet152V2.h5')\n# 모델을 저장할 때 사용되는 콜백함수\n\nes = EarlyStopping(verbose=1, patience=2)\n# 모델이 더이상 학습을 못할 경우(loss, metric 등의 개선이 없을 경우), 학습 도중 미리 학습을 종료시키는 콜백함수\n\n## 참고 링크 : https://deep-deep-deep.tistory.com/55","metadata":{"_uuid":"e30fc8b6-07f1-4011-b5c8-8321614ab10a","_cell_guid":"ff9cf9de-71da-45b6-b180-70e12cda9cf1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:54.863626Z","iopub.execute_input":"2021-06-20T07:58:54.86404Z","iopub.status.idle":"2021-06-20T07:58:54.87401Z","shell.execute_reply.started":"2021-06-20T07:58:54.863999Z","shell.execute_reply":"2021-06-20T07:58:54.873165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='Adam', loss=\"categorical_crossentropy\", metrics=METRICS)\n# optimizer : 손실함수 기반으로 어떻게 네트워크를 업데이트할 지 결정해줌\n# loss : 손실 함수, 입력데이터와 출력데이터의 일치여부를 평가해주는 함수\n# metrics : 평가 기준 (학습이 제대로 되고 있는지 살펴볼 수 있음)","metadata":{"_uuid":"57e90d36-10f5-48f4-8362-0bf260cc615a","_cell_guid":"c9af97d7-1124-45de-b064-6ed1473c487b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:56.301937Z","iopub.execute_input":"2021-06-20T07:58:56.302319Z","iopub.status.idle":"2021-06-20T07:58:56.381785Z","shell.execute_reply.started":"2021-06-20T07:58:56.302284Z","shell.execute_reply":"2021-06-20T07:58:56.380947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size +1\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size +1\n# 1 epoch당 몇번 돌아야 하는지","metadata":{"_uuid":"de052608-bdcd-47dd-861e-b12d5c95c376","_cell_guid":"22873bb5-15f7-4255-b3aa-1c62864c6457","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:57.466997Z","iopub.execute_input":"2021-06-20T07:58:57.467415Z","iopub.status.idle":"2021-06-20T07:58:57.475837Z","shell.execute_reply.started":"2021-06-20T07:58:57.467375Z","shell.execute_reply":"2021-06-20T07:58:57.474852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(STEP_SIZE_TRAIN)\nprint(STEP_SIZE_VALID)","metadata":{"_uuid":"c5fcad79-1edf-4236-ad85-b411d646a589","_cell_guid":"d9197175-2eb4-45d3-814f-b35c3e07684d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:58.401109Z","iopub.execute_input":"2021-06-20T07:58:58.401668Z","iopub.status.idle":"2021-06-20T07:58:58.412277Z","shell.execute_reply.started":"2021-06-20T07:58:58.401518Z","shell.execute_reply":"2021-06-20T07:58:58.411364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model training","metadata":{}},{"cell_type":"code","source":"%time\nhistory = model.fit_generator(generator=train_generator,steps_per_epoch=STEP_SIZE_TRAIN,validation_data=valid_generator,validation_steps=STEP_SIZE_VALID,epochs=10,callbacks=[lrd,mcp,es])","metadata":{"_uuid":"0db33ed9-6e28-4e11-b785-c0138af5ee45","_cell_guid":"abdf6ff9-5c6e-4415-a804-bdb5bbfbd1b6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:58:59.890923Z","iopub.execute_input":"2021-06-20T07:58:59.891363Z","iopub.status.idle":"2021-06-20T08:12:19.357559Z","shell.execute_reply.started":"2021-06-20T07:58:59.891316Z","shell.execute_reply":"2021-06-20T08:12:19.356647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"complete_datagen = ImageDataGenerator(rescale=1./255)\ncomplete_generator = complete_datagen.flow_from_dataframe( \n dataframe=train,\n directory = \"../input/preprocess-nb/preprocess/1/\",\n x_col=\"id_code\",\n target_size=(224, 224),\n batch_size=1,\n shuffle=False,\n class_mode=None)","metadata":{"_uuid":"a3d4c438-b31e-409b-bbca-6b5842a05199","_cell_guid":"f438225d-a0b4-4b24-9801-d225a52f0bce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T08:12:53.326175Z","iopub.execute_input":"2021-06-20T08:12:53.326604Z","iopub.status.idle":"2021-06-20T08:12:53.617862Z","shell.execute_reply.started":"2021-06-20T08:12:53.326561Z","shell.execute_reply":"2021-06-20T08:12:53.609614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_COMPLETE = complete_generator.n//complete_generator.batch_size\ntrain_preds = model.predict_generator(complete_generator, steps=STEP_SIZE_COMPLETE)\ntrain_preds = [np.argmax(pred) for pred in train_preds]","metadata":{"_uuid":"d002771d-ff42-4759-bdaf-6c21f16fe1b7","_cell_guid":"d53ccb6b-2488-40b0-841f-62966c730fab","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T08:12:55.16295Z","iopub.execute_input":"2021-06-20T08:12:55.163376Z","iopub.status.idle":"2021-06-20T08:15:00.539787Z","shell.execute_reply.started":"2021-06-20T08:12:55.163336Z","shell.execute_reply":"2021-06-20T08:15:00.536848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T08:51:09.857225Z","iopub.execute_input":"2021-06-20T08:51:09.857569Z","iopub.status.idle":"2021-06-20T08:51:09.86598Z","shell.execute_reply.started":"2021-06-20T08:51:09.857515Z","shell.execute_reply":"2021-06-20T08:51:09.865057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# confusion matrix","metadata":{}},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ncnf_matrix = confusion_matrix(train['diagnosis'].astype('int'), train_preds)\ncnf_matrix_norm = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\ndf_cm = pd.DataFrame(cnf_matrix_norm, index=labels, columns=labels)\nprint(df_cm.describe().T)\nplt.figure(figsize=(15, 8))\nsns.heatmap(df_cm, annot=True, fmt='.2f')\nplt.show()","metadata":{"_uuid":"ce4fe80e-8dd4-4231-a23b-91a643769c74","_cell_guid":"7d39feae-96b1-4fd8-a76b-7e2efe821034","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T08:51:12.573235Z","iopub.execute_input":"2021-06-20T08:51:12.57358Z","iopub.status.idle":"2021-06-20T08:51:12.909614Z","shell.execute_reply.started":"2021-06-20T08:51:12.573538Z","shell.execute_reply":"2021-06-20T08:51:12.908854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cm.head(5)","metadata":{"_uuid":"b6e80faa-e162-4471-bd6a-beb684955f64","_cell_guid":"7bde561d-b682-4c75-88ef-d3f803021375","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T07:53:11.528868Z","iopub.status.idle":"2021-06-20T07:53:11.533876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training set에 대한 성능평가","metadata":{}},{"cell_type":"code","source":"get_acc = history.history['accuracy']\nvalue_acc = history.history['val_accuracy']\nget_loss = history.history['loss']\nvalidation_loss = history.history['val_loss']\nget_auc = history.history['auc']\nvalidation_auc = history.history['val_auc']\nget_f1 = history.history['f1_score']\nvalidation_f1 = history.history['val_f1_score']\n\nepochs = range(len(get_acc))\nplt.plot(epochs, get_acc, 'r', label='Accuracy of Training data')\nplt.plot(epochs, value_acc, 'b', label='Accuracy of Validation data')\nplt.title('Training vs validation accuracy')\nplt.legend(loc=0)\nplt.figure()\nplt.show()","metadata":{"_uuid":"8f7469fd-1599-4989-92aa-31352d5c60d7","_cell_guid":"d58f380d-e259-434b-9d4b-ba9b1061fee4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:37.389411Z","iopub.execute_input":"2021-06-20T06:52:37.389816Z","iopub.status.idle":"2021-06-20T06:52:37.588267Z","shell.execute_reply.started":"2021-06-20T06:52:37.38978Z","shell.execute_reply":"2021-06-20T06:52:37.587306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(len(get_auc))\nplt.plot(epochs, get_auc, 'r', label='auc of Training data')\nplt.plot(epochs, validation_auc, 'b', label='auc of Validation data')\nplt.title('Training vs validation auc')\nplt.legend(loc=0)\nplt.figure()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:52:39.95026Z","iopub.execute_input":"2021-06-20T06:52:39.950625Z","iopub.status.idle":"2021-06-20T06:52:40.169937Z","shell.execute_reply.started":"2021-06-20T06:52:39.950571Z","shell.execute_reply":"2021-06-20T06:52:40.168953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(len(get_f1))\nplt.plot(epochs, get_f1, 'r', label='f1 score of Training data')\nplt.plot(epochs, validation_f1, 'b', label='f1 score of Validation data')\nplt.title('Training vs validation f1 score')\nplt.legend(loc=0)\nplt.figure()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:52:42.623262Z","iopub.execute_input":"2021-06-20T06:52:42.62361Z","iopub.status.idle":"2021-06-20T06:52:42.831622Z","shell.execute_reply.started":"2021-06-20T06:52:42.623559Z","shell.execute_reply":"2021-06-20T06:52:42.830665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(len(get_loss))\nplt.plot(epochs, get_loss, 'r', label='Loss of Training data')\nplt.plot(epochs, validation_loss, 'b', label='Loss of Validation data')\nplt.title('Training vs validation loss')\nplt.legend(loc=0)\nplt.figure()\nplt.show()","metadata":{"_uuid":"409614e6-73de-4d3b-8165-9c976807011e","_cell_guid":"689aec04-00ef-4dc2-8b92-efe265cf2f80","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:45.354692Z","iopub.execute_input":"2021-06-20T06:52:45.355037Z","iopub.status.idle":"2021-06-20T06:52:45.559675Z","shell.execute_reply.started":"2021-06-20T06:52:45.355005Z","shell.execute_reply":"2021-06-20T06:52:45.558743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds, train['diagnosis'].astype('int'), weights='quadratic'))","metadata":{"_uuid":"fb70f80c-e214-4e5d-9b84-09d6726e9627","_cell_guid":"f019c12f-7474-4597-9e9f-a757750e7394","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:48.084198Z","iopub.execute_input":"2021-06-20T06:52:48.084536Z","iopub.status.idle":"2021-06-20T06:52:48.094743Z","shell.execute_reply.started":"2021-06-20T06:52:48.084502Z","shell.execute_reply":"2021-06-20T06:52:48.093762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test set prediction","metadata":{}},{"cell_type":"code","source":"test_generator.reset()\nSTEP_SIZE_TEST = test_generator.n//test_generator.batch_size +1\npreds = model.predict_generator(test_generator, steps=STEP_SIZE_TEST)\npredictions = [np.argmax(pred) for pred in preds]\npredictions[:10]","metadata":{"_uuid":"41ed876d-c6cd-49c2-a4b0-80aadf3a0e1b","_cell_guid":"80a38e93-9eaa-4d7b-8b0d-d5c8c5f35e6c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:50.59008Z","iopub.execute_input":"2021-06-20T06:52:50.590416Z","iopub.status.idle":"2021-06-20T06:52:52.432667Z","shell.execute_reply.started":"2021-06-20T06:52:50.590382Z","shell.execute_reply":"2021-06-20T06:52:52.431861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = test_generator.filenames\nresults = pd.DataFrame(['id_code',filenames, 'diagnosis',predictions])\nresults.to_csv('submission.csv',index=False)\nresults.head(5).T","metadata":{"_uuid":"25266ab4-a7c9-4585-8d9e-324045982419","_cell_guid":"6a6a88ff-2fb3-4111-8231-dec0cc4fceba","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:55.474847Z","iopub.execute_input":"2021-06-20T06:52:55.475214Z","iopub.status.idle":"2021-06-20T06:52:55.497974Z","shell.execute_reply.started":"2021-06-20T06:52:55.475181Z","shell.execute_reply":"2021-06-20T06:52:55.497169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(filenames))\nprint(len(predictions))","metadata":{"_uuid":"fb992a72-1262-44ce-9537-cf7c353fb4f4","_cell_guid":"9f55bd15-0c4d-4817-917f-034f0bcc64a5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:52:58.265534Z","iopub.execute_input":"2021-06-20T06:52:58.265906Z","iopub.status.idle":"2021-06-20T06:52:58.273547Z","shell.execute_reply.started":"2021-06-20T06:52:58.265871Z","shell.execute_reply":"2021-06-20T06:52:58.272576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames=test_generator.filenames\nresults=pd.DataFrame({\"id_code\":filenames,\n \"diagnosis\":predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])\nresults.to_csv(\"submission.csv\",index=False)","metadata":{"_uuid":"abe32513-8516-4842-b216-3daca2ad1473","_cell_guid":"26cdf7aa-ab4e-4902-a564-ff7536b26f02","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:53:01.30872Z","iopub.execute_input":"2021-06-20T06:53:01.30904Z","iopub.status.idle":"2021-06-20T06:53:01.319674Z","shell.execute_reply.started":"2021-06-20T06:53:01.30901Z","shell.execute_reply":"2021-06-20T06:53:01.318697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test set의 실제 diagnosis value","metadata":{}},{"cell_type":"code","source":"test.iloc[:,1].astype('int').hist()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:07.046178Z","iopub.execute_input":"2021-06-20T06:53:07.046503Z","iopub.status.idle":"2021-06-20T06:53:07.236234Z","shell.execute_reply.started":"2021-06-20T06:53:07.04647Z","shell.execute_reply":"2021-06-20T06:53:07.235253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prediction value","metadata":{}},{"cell_type":"code","source":"results.hist()","metadata":{"_uuid":"78160901-4fc4-4716-bacb-19ea6c6e1a96","_cell_guid":"1b7cfa5d-c52a-4e0e-af87-0455b32b5706","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:53:09.70825Z","iopub.execute_input":"2021-06-20T06:53:09.708574Z","iopub.status.idle":"2021-06-20T06:53:09.901475Z","shell.execute_reply.started":"2021-06-20T06:53:09.708542Z","shell.execute_reply":"2021-06-20T06:53:09.900506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test set confusion matrix","metadata":{}},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ncnf_matrix = confusion_matrix(test['diagnosis'].astype('int'), predictions)\ncnf_matrix_norm = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\ndf_cm = pd.DataFrame(cnf_matrix_norm, index=labels, columns=labels)\nprint(df_cm.describe().T)\nplt.figure(figsize=(15, 8))\nsns.heatmap(df_cm, annot=True, fmt='.2f')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:12.637343Z","iopub.execute_input":"2021-06-20T06:53:12.637671Z","iopub.status.idle":"2021-06-20T06:53:12.978921Z","shell.execute_reply.started":"2021-06-20T06:53:12.63764Z","shell.execute_reply":"2021-06-20T06:53:12.978161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test set에 대한 성능평가","metadata":{}},{"cell_type":"code","source":"print(\"Test Cohen Kappa score: %.3f\" % cohen_kappa_score(predictions, test['diagnosis'].astype('int'), weights='quadratic'))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:28.33125Z","iopub.execute_input":"2021-06-20T06:53:28.331653Z","iopub.status.idle":"2021-06-20T06:53:28.340724Z","shell.execute_reply.started":"2021-06-20T06:53:28.331606Z","shell.execute_reply":"2021-06-20T06:53:28.339519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\n\nlist1=test['diagnosis'].astype('int').tolist()\nacc=accuracy_score(list1, predictions)\nrec=recall_score(list1, predictions,average='macro')\npre=precision_score(list1, predictions,average='macro')\nf1=f1_score(list1, predictions, average='macro')\nprint(acc,rec,pre,f1)","metadata":{"_uuid":"abbb62e4-356e-4616-a741-188a9d51f1de","_cell_guid":"e84c9244-bd28-4ea6-a70c-5b77009311a9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-20T06:53:31.895572Z","iopub.execute_input":"2021-06-20T06:53:31.896006Z","iopub.status.idle":"2021-06-20T06:53:31.911518Z","shell.execute_reply.started":"2021-06-20T06:53:31.895965Z","shell.execute_reply":"2021-06-20T06:53:31.910235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = ['accuracy', 'recall', 'precision', 'f1_score']\nscore = [acc,rec,pre,f1]\ndf = pd.DataFrame({'evaluation':name,'score':score})\n\nplt.figure(figsize=(8, 6))\nsplot=sns.barplot(x=\"evaluation\",y=\"score\",data=df)\nfor p in splot.patches:\n splot.annotate(format(p.get_height(), '.3f'), \n (p.get_x() + p.get_width() / 2., p.get_height()), \n ha = 'center', va = 'center', \n size=15,\n xytext = (0, -12), \n textcoords = 'offset points')\nplt.xlabel(\"evaluation\", size=14)\nplt.ylabel(\"score\", size=14)\n#plt.savefig(\"add_annotation_to_bars_in_barplot_Seaborn_Python.png\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:34.565141Z","iopub.execute_input":"2021-06-20T06:53:34.565485Z","iopub.status.idle":"2021-06-20T06:53:34.731297Z","shell.execute_reply.started":"2021-06-20T06:53:34.565436Z","shell.execute_reply":"2021-06-20T06:53:34.730516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:01:47.266896Z","iopub.execute_input":"2021-06-14T17:01:47.267222Z","iopub.status.idle":"2021-06-14T17:01:47.912059Z","shell.execute_reply.started":"2021-06-14T17:01:47.267193Z","shell.execute_reply":"2021-06-14T17:01:47.911107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}