{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport matplotlib.pyplot as plt\nfrom functools import partial\n\n\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\n# import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Data:\n**TPUs** will read the data from **Google Cloud Storage**(GCS), so we need to specify the training data path in GCS","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n!gsutil ls $GCS_DS_PATH # list the bucket","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We will see that we set a bigger batch size (128) because the data is imbalanced**","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [1024, 1024]\nIMAGE_RESIZE = [256, 256]\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\nprint(BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES, VALIDATION_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords/train*.tfrec'),\n    test_size=0.2, random_state=42\n)\n\nprint(len(TRAINING_FILENAMES))\nprint(len(VALIDATION_FILENAMES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords/test*.tfrec')\nprint(len(TEST_FILENAMES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dropout(image, DIM=IMAGE_RESIZE[0], PROBABILITY = 0.5, CT = 8, SZ = 0.2):\n    # input - one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n\n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n    if (P == 0)|(CT == 0)|(SZ == 0): return image\n\n    for k in range( CT ):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation_pipeline(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    image = dropout(image, DIM=IMAGE_RESIZE[0], PROBABILITY = 0.5, CT = 8, SZ = 0.2)\n    \n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation_pipeline_val(image, label):\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation_pipeline_test(image, name):\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, name\n# def augmentation_pipeline_test(image):\n#     image = tf.image.resize(image, IMAGE_RESIZE)\n#     return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_dataset(file):\n#     dataset = load_dataset(file, labeled=True)\n#     dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n#     dataset = dataset.repeat()\n#     dataset = dataset.shuffle(2048)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     dataset = dataset.prefetch(AUTOTUNE)\n#     return dataset\n\n\n# def get_test_dataset(file):\n#     dataset = load_dataset(file)\n#     dataset = dataset.map(augmentation_pipeline_test, num_parallel_calls=AUTOTUNE)\n# #     dataset = dataset.repeat()\n# #     dataset = dataset.shuffle(2048)\n# #     dataset = dataset.batch(BATCH_SIZE)\n# #     dataset = dataset.prefetch(AUTOTUNE)\n#     return dataset\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(augmentation_pipeline_val, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered) \n    dataset = dataset.map(augmentation_pipeline_test, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_batch, label_batch = next(iter(train_dataset))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch(image_batch, label_batch):\n    plt.figure(figsize=(10,10))\n    for n in range(16):\n        ax = plt.subplot(4,4,n+1)\n        plt.imshow(image_batch[n])\n        if label_batch[n]:\n            plt.title(\"MALIGNANT\")\n        else:\n            plt.title(\"BENIGN\")\n        plt.axis(\"off\")\n        \nshow_batch(image_batch.numpy(), label_batch.numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ================= Build Model ======================\n","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers.experimental import preprocessing\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers\n# from focal_loss import BinaryFocalLoss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Training_Model(model_name, IMG_SIZE, NUM_CHANNELS, Dropout_rate):\n    inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n    \n    if model_name == 'VGG19':\n        base_model = tf.keras.applications.vgg19.VGG19(input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n    \n    if model_name == 'ResNet152V2':\n        base_model = tf.keras.applications.resnet_v2.ResNet152V2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'InceptionV3':\n        base_model = tf.keras.applications.InceptionV3( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'Xception':\n        base_model=tf.keras.applications.Xception( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'EfficientNetB2':\n        base_model=tf.keras.applications.EfficientNetB2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n        \n    if model_name == 'EfficientNetB3':\n        base_model=tf.keras.applications.EfficientNetB3( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'EfficientNetB4':\n        base_model=tf.keras.applications.EfficientNetB4( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'InceptionResNetV2':\n        base_model=tf.keras.applications.InceptionResNetV2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'DenseNet201':\n        base_model=tf.keras.applications.DenseNet201( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'MobileNetV2':\n        base_model=tf.keras.applications.MobileNetV2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n        \n    if model_name == 'ResNet101V2':\n        base_model=tf.keras.applications.ResNet101V2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        base_model.trainable = False\n        x = base_model.output\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(1000, activation='relu')(x)\n        x = layers.Dropout(Dropout_rate)(x)\n        x = layers.Dense(1, activation='sigmoid')(x)\n    \n        \n    model = models.Model(inputs=base_model.input, outputs=x)\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['AUC']) \n#     model.compile(optimizer='adam', loss=BinaryFocalLoss(gamma=2), metrics=['AUC'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ==================== Training =========================","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    model1 = Training_Model(model_name='ResNet101V2', IMG_SIZE=IMAGE_RESIZE[0], NUM_CHANNELS=3, Dropout_rate=0.4)\n    model2 = Training_Model(model_name='DenseNet201', IMG_SIZE=IMAGE_RESIZE[0], NUM_CHANNELS=3, Dropout_rate=0.4)\n    model3 = Training_Model(model_name='MobileNetV2', IMG_SIZE=IMAGE_RESIZE[0], NUM_CHANNELS=3, Dropout_rate=0.4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 5\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\n# checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"ResNet_model.h5\", save_best_only=True)\n# early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)\n\nclass_weight = {0: 0.5, 1: 28.0}\n\nhistory1 = model1.fit(\n    train_dataset,\n    epochs=epochs,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    validation_data=valid_dataset,\n    validation_steps=VALID_STEPS\n#     callbacks=[checkpoint_cb, early_stopping_cb],\n#     class_weight=class_weight\n\n)\nmodel1.save('model1.hdf5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2 = model2.fit(\n    train_dataset,\n    epochs=epochs,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    validation_data=valid_dataset,\n    validation_steps=VALID_STEPS,\n#     callbacks=[checkpoint_cb, early_stopping_cb],\n    class_weight=class_weight\n\n)\n\nmodel2.save('model2.hdf5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3 = model3.fit(\n    train_dataset,\n    epochs=epochs,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    validation_data=valid_dataset,\n    validation_steps=VALID_STEPS,\n#     callbacks=[checkpoint_cb, early_stopping_cb],\n    class_weight=class_weight\n\n)\nmodel3.save('model3.hdf5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## =================== Prediction ===================","metadata":{}},{"cell_type":"code","source":"test_ds = get_test_dataset(TEST_FILENAMES)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\n\nmodel_list = [model1, model2, model3]\n\n# probabilities1 = model1.predict(test_images_ds)\n# probabilities2 = model2.predict(test_images_ds)\n# probabilities3 = model3.predict(test_images_ds)\nens_probabilities = [model.predict(test_images_ds) for model in model_list]\n\n\nprint(\"========================  Done  ============================\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average the predictions of models\n\naverage_prob = np.sum(ens_probabilities, axis=0)/len(model_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weight the prediction of models\n\n\nweights = [0.4, 0.25, 0.35]\n\n#Use tensordot to sum the products of all elements over specified axes.\nweighted_prob = np.tensordot(ens_probabilities, weights, axes=((0),(0)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\nprint(\"========================  Done  ============================\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df_weighted = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(weighted_prob)})\npred_df_av = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(average_prob)})\n\npred_df_weighted.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df_weighted.to_csv('submission_weighted.csv', index=False)\npred_df_av.to_csv('submission_av.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}}]}