{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet\ntf.compat.v1.disable_eager_execution()","metadata":{"execution":{"iopub.status.busy":"2023-10-16T04:03:07.536468Z","iopub.execute_input":"2023-10-16T04:03:07.536887Z","iopub.status.idle":"2023-10-16T04:03:17.645252Z","shell.execute_reply.started":"2023-10-16T04:03:07.536858Z","shell.execute_reply":"2023-10-16T04:03:17.643867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\nfrom efficientnet.keras import EfficientNetB5\nimport cv2\nimport time\nimport scipy as sp\nimport numpy as np\nimport random as rn\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom functools import partial\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport os\nimport sys\nimport keras\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import backend as K\nfrom keras.activations import elu\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras.layers import Layer, InputSpec\nfrom keras.utils import get_custom_objects\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D\nfrom keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import cohen_kappa_score\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nKAGGLE_DIR = '/kaggle/input/aptos2019-blindness-detection/'\nTRAIN_DF_PATH = KAGGLE_DIR + \"train.csv\"\nTEST_DF_PATH = KAGGLE_DIR + 'test.csv'\nTRAIN_IMG_PATH = KAGGLE_DIR + \"train_images/\"\nTEST_IMG_PATH = KAGGLE_DIR + 'test_images/'\nSAVED_MODEL_NAME = 'effnet_modelB5.h5'\nseed = 1234\n\nrn.seed(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nt_start = time.time()\nprint(\"Image IDs and Labels (TRAIN)\")\ntrain_df = pd.read_csv(TRAIN_DF_PATH)\ntrain_df['id_code'] = train_df['id_code'] + \".png\"\nprint(f\"Training images: {train_df.shape[0]}\")\ndisplay(train_df.head())\n\nprint(\"Image IDs (TEST)\")\ntest_df = pd.read_csv(TEST_DF_PATH)\ntest_df['id_code'] = test_df['id_code'] + \".png\"\nprint(f\"Testing Images: {test_df.shape[0]}\")\ndisplay(test_df.head())\n\nIMG_WIDTH = 456\nIMG_HEIGHT = 456\nCHANNELS = 3\n\ndef get_preds_and_labels(model, generator):\n    preds = []\n    labels = []\n    for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):\n        x, y = next(generator)\n        preds.append(model.predict(x))\n        labels.append(y)\n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()\n\nclass Metrics(Callback):\n\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        y_pred, labels = get_preds_and_labels(model, val_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n        self.model.save(SAVED_MODEL_NAME)\n        return\n\ntrain_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\",\nfigsize=(12,5),\nrot=0)\nplt.title(\"Label Distribution (Training Set)\",\nweight='bold',\nfontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);\ntrain_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\",\nfigsize=(12,5), rot=0)\n\n# Function for Cropping the image\ndef crop_image_from_gray(img, tol=7):\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n    if (check_shape == 0):\n        return img\n    else:\n        img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n        img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n        img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n        img = np.stack([img1,img2,img3],axis=-1)\n        return img\n    \n#Preprocessing the image\ndef preprocess_image(image, sigmaX=10):\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_WIDTH, IMG_HEIGHT))\n    image = cv2.addWeighted(image,4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -\n    4, 128)\n    return image\n\nfig, ax = plt.subplots(1, 5, figsize=(15, 6))\nfor i in range(5):\n    sample = train_df[train_df['diagnosis'] == i].sample(1)\n    image_name = sample['id_code'].item()\n    X = preprocess_image(cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\"))\n    ax[i].set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\",\n    weight='bold', fontsize=10)\n    ax[i].axis('off')\n    ax[i].imshow(X);\nBATCH_SIZE = 4\n\ntrain_datagen = ImageDataGenerator(rotation_range=360,\n                                horizontal_flip=True,\n                                vertical_flip=True,\n                                validation_split=0.15,\n                                preprocessing_function=preprocess_image,\n                                rescale=1 / 128.)\ntrain_generator = train_datagen.flow_from_dataframe(train_df,\n                                                    x_col='id_code',\n                                                    y_col='diagnosis',\n                                                    directory = TRAIN_IMG_PATH,\n                                                    target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                    batch_size=BATCH_SIZE,\n                                                    class_mode='other',\n                                                    subset='training')\n\nval_generator = train_datagen.flow_from_dataframe(train_df,\n                                                    x_col='id_code',\n                                                    y_col='diagnosis',\n                                                    directory = TRAIN_IMG_PATH,\n                                                    target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                    batch_size=BATCH_SIZE,\n                                                    class_mode='other',\n                                                    subset='validation')\nclass RAdam(keras.optimizers.Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n        \n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            decay_steps = self.total_steps - warmup_steps\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n    \n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n    \nclass GroupNormalization(Layer):\n    def __init__(self,\n            groups=32,\n            axis=-1,\n            epsilon=1e-5,\n            center=True,\n            scale=True,\n            beta_initializer='zeros',\n            gamma_initializer='ones',\n            beta_regularizer=None,\n            gamma_regularizer=None,\n            beta_constraint=None,\n            gamma_constraint=None,\n            **kwargs):\n        super(GroupNormalization, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.groups = groups\n        self.axis = axis\n        self.epsilon = epsilon\n        self.center = center\n        self.scale = scale\n        self.beta_initializer = initializers.get(beta_initializer)\n        self.gamma_initializer = initializers.get(gamma_initializer)\n        self.beta_regularizer = regularizers.get(beta_regularizer)\n        self.gamma_regularizer = regularizers.get(gamma_regularizer)\n        self.beta_constraint = constraints.get(beta_constraint)\n        self.gamma_constraint = constraints.get(gamma_constraint)\n\n    def build(self, input_shape):\n        dim = input_shape[self.axis]\n        if dim is None:\n            raise ValueError('Axis ' + str(self.axis) + ' of '\n            'input tensor should have a defined dimension '\n            'but the layer received an input with shape ' +\n            str(input_shape) + '.')\n        if dim < self.groups:\n            raise ValueError('Number of groups (' + str(self.groups) + ') cannot be '\n            'more than the number of channels (' +\n            str(dim) + ').')\n        if dim % self.groups != 0:\n            raise ValueError('Number of groups (' + str(self.groups) + ') must be a '\n            'multiple of the number of channels (' +\n            str(dim) + ').')\n            self.input_spec = InputSpec(ndim=len(input_shape),\n            axes={self.axis: dim})\n            shape = (dim,)\n        if self.scale:\n            self.gamma = self.add_weight(shape=shape,\n            name='gamma',\n            initializer=self.gamma_initializer,\n            regularizer=self.gamma_regularizer,\n            constraint=self.gamma_constraint)\n        else:\n            self.gamma = None\n        if self.center:\n            self.beta = self.add_weight(shape=shape,\n            name='beta',\n            initializer=self.beta_initializer,\n            regularizer=self.beta_regularizer,\n            constraint=self.beta_constraint)\n        else:\n            self.beta = None\n            self.built = True\n\n    def call(self, inputs, **kwargs):\n        input_shape = K.int_shape(inputs)\n        tensor_input_shape = K.shape(inputs)\n        # Prepare broadcasting shape.\n        reduction_axes = list(range(len(input_shape)))\n        del reduction_axes[self.axis]\n        broadcast_shape = [1] * len(input_shape)\n        broadcast_shape[self.axis] = input_shape[self.axis] // self.groups\n        broadcast_shape.insert(1, self.groups)\n        reshape_group_shape = K.shape(inputs)\n        group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]\n        group_axes[self.axis] = input_shape[self.axis] // self.groups\n        group_axes.insert(1, self.groups)\n        group_shape = [group_axes[0], self.groups] + group_axes[2:]\n        group_shape = K.stack(group_shape)\n        inputs = K.reshape(inputs, group_shape)\n        group_reduction_axes = list(range(len(group_axes))) \n        group_reduction_axes = group_reduction_axes[2:]\n        mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)\n        variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)\n        inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))\n        inputs = K.reshape(inputs, group_shape)\n        outputs = inputs\n        if self.scale:\n            broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n            outputs = outputs * broadcast_gamma\n        if self.center:\n            broadcast_beta = K.reshape(self.beta, broadcast_shape)\n            outputs = outputs + broadcast_beta\n            outputs = K.reshape(outputs, tensor_input_shape)\n        return outputs\n\n    def get_config(self):\n        config = {\n        'groups': self.groups,\n        'axis': self.axis,\n        'epsilon': self.epsilon,\n        'center': self.center,\n        'scale': self.scale,\n        'beta_initializer': initializers.serialize(self.beta_initializer),\n        'gamma_initializer': initializers.serialize(self.gamma_initializer),\n        'beta_regularizer': regularizers.serialize(self.beta_regularizer),\n        'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),\n        'beta_constraint': constraints.serialize(self.beta_constraint),\n        'gamma_constraint': constraints.serialize(self.gamma_constraint)\n        }\n        base_config = super(GroupNormalization, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n    \neffnet = EfficientNetB5(weights=None,\ninclude_top=False,\ninput_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS))\neffnet.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')\nfor i, layer in enumerate(effnet.layers):\n    if \"batch_normalization\" in layer.name:\n        effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001)\n\ndef build_model():\n\n    model = Sequential()\n    model.add(effnet)\n    model.add(GlobalAveragePooling2D())\n    model.add(Dropout(0.5))\n    model.add(Dense(5, activation=elu))\n    model.add(Dense(1, activation=\"linear\"))\n    model.compile(loss='mse',\n    optimizer=RAdam(lr=0.00005),\n    metrics=['mse', 'acc'])\n    print(model.summary())\n    return model\n\nmodel = build_model()\nkappa_metrics = Metrics()\n\n# Monitor MSE to avoid overfitting and save best model\nes = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)\nrlr = ReduceLROnPlateau(monitor='val_loss',\n        factor=0.5,\n        patience=4,\n        verbose=1,\n        mode='auto',\n        epsilon=0.0001)\n\n# Begin training\nmodel.fit(train_generator,\n    steps_per_epoch=train_generator.samples // BATCH_SIZE,\n    epochs=35,\n    validation_data=val_generator,\n    validation_steps = val_generator.samples // BATCH_SIZE,\n    callbacks=[kappa_metrics, es, rlr])\nmodel.load_weights('../input/trainmodel2/effnet_original.h5')\nhistory_df = pd.DataFrame(model.history.history)\nhistory_df[['loss', 'val_loss']].plot(figsize=(12,5))\nplt.title(\"Loss (MSE)\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss (MSE)\")\nhistory_df[['acc', 'val_acc']].plot(figsize=(12,5))\nplt.title(\"Accuracy\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"% Accuracy\");\ny_train_preds, train_labels = get_preds_and_labels(model, train_generator)\ny_train_preds = np.rint(y_train_preds).astype(np.uint8).clip(0, 4)\n# Calculate score\ntrain_score = cohen_kappa_score(train_labels, y_train_preds, weights=\"quadratic\")\n# Calculate QWK on validation set\ny_val_preds, val_labels = get_preds_and_labels(model, val_generator)\ny_val_preds = np.rint(y_val_preds).astype(np.uint8).clip(0, 4)\n# Calculate score\nval_score = cohen_kappa_score(val_labels, y_val_preds, weights=\"quadratic\")\nprint(f\"The Training Cohen Kappa Score is: {round(train_score, 5)}\")\nprint(f\"The Validation Cohen Kappa Score is: {round(val_score, 5)}\")\n\nclass OptimizedRounder(object):\n\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        ll = cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        \"\"\"\n        Return the optimized coefficients\n        \"\"\"\n        return self.coef_['x']\n\n# Optimize on validation data and evaluate again\ny_val_preds, val_labels = get_preds_and_labels(model, val_generator)\noptR = OptimizedRounder()\noptR.fit(y_val_preds, val_labels)\ncoefficients = optR.coefficients()\nopt_val_predictions = optR.predict(y_val_preds, coefficients)\nnew_val_score = cohen_kappa_score(val_labels, opt_val_predictions,\nweights=\"quadratic\")\ntest_df['diagnosis'] = np.zeros(test_df.shape[0])\n\n# For preprocessing test images\ntest_generator = ImageDataGenerator(preprocessing_function=preprocess_image,\nrescale=1 / 128.).flow_from_dataframe(test_df,\nx_col='id_code', y_col='diagnosis',directory=TEST_IMG_PATH,\ntarget_size=(IMG_WIDTH, IMG_HEIGHT), batch_size=BATCH_SIZE,\nclass_mode='other', shuffle=False)\ntrain_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\",figsize=(12,5),\nrot=0)\n\nplt.title(\"Label Distribution (Training Set)\", weight='bold', fontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);\nt_finish = time.time()\ntotal_time = round((t_finish-t_start) / 3600, 4)\nprint('Kernel runtime = {} hours ({} minutes)'.format(total_time, int(total_time*60)))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-16T04:03:17.648854Z","iopub.execute_input":"2023-10-16T04:03:17.649343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-10-16T03:59:12.923989Z","iopub.status.idle":"2023-10-16T03:59:12.924780Z","shell.execute_reply.started":"2023-10-16T03:59:12.924554Z","shell.execute_reply":"2023-10-16T03:59:12.924594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}