{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Histopathic Cancer Detection (HCD)\n### Taylor Kern","metadata":{}},{"cell_type":"markdown","source":"# Prepare Enviornment","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport pickle\nimport os\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import * \n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import backend as k\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' ","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:33.770699Z","iopub.execute_input":"2022-04-14T19:15:33.77103Z","iopub.status.idle":"2022-04-14T19:15:39.860017Z","shell.execute_reply.started":"2022-04-14T19:15:33.770959Z","shell.execute_reply":"2022-04-14T19:15:39.859104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def merge_history(hlist):\n    history = {}\n    for k in hlist[0].history.keys():\n        history[k] = sum([h.history[k] for h in hlist], [])\n    return history\n\ndef vis_training(h, start=1):\n    epoch_range = range(start, len(h['loss'])+1)\n    s = slice(start-1, None)\n\n    plt.figure(figsize=[14,4])\n\n    n = int(len(h.keys()) / 2)\n\n    for i in range(n):\n        k = list(h.keys())[i]\n        plt.subplot(1,n,i+1)\n        plt.plot(epoch_range, h[k][s], label='Training')\n        plt.plot(epoch_range, h['val_' + k][s], label='Validation')\n        plt.xlabel('Epoch'); plt.ylabel(k); plt.title(k)\n        plt.grid()\n        plt.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:45.293491Z","iopub.execute_input":"2022-04-14T19:15:45.293877Z","iopub.status.idle":"2022-04-14T19:15:45.3042Z","shell.execute_reply.started":"2022-04-14T19:15:45.293843Z","shell.execute_reply":"2022-04-14T19:15:45.303451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Training DataFrame","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv', dtype=str)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:45.636798Z","iopub.execute_input":"2022-04-14T19:15:45.637052Z","iopub.status.idle":"2022-04-14T19:15:45.98218Z","shell.execute_reply.started":"2022-04-14T19:15:45.637025Z","shell.execute_reply":"2022-04-14T19:15:45.981401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:45.983843Z","iopub.execute_input":"2022-04-14T19:15:45.984247Z","iopub.status.idle":"2022-04-14T19:15:46.001032Z","shell.execute_reply.started":"2022-04-14T19:15:45.984209Z","shell.execute_reply":"2022-04-14T19:15:46.000305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.id = train.id + '.tif'","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:46.002267Z","iopub.execute_input":"2022-04-14T19:15:46.002539Z","iopub.status.idle":"2022-04-14T19:15:46.049877Z","shell.execute_reply.started":"2022-04-14T19:15:46.002505Z","shell.execute_reply":"2022-04-14T19:15:46.049125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:46.16616Z","iopub.execute_input":"2022-04-14T19:15:46.16636Z","iopub.status.idle":"2022-04-14T19:15:46.174473Z","shell.execute_reply.started":"2022-04-14T19:15:46.166336Z","shell.execute_reply":"2022-04-14T19:15:46.173731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Distribution","metadata":{}},{"cell_type":"code","source":"(train.label.value_counts() / len(train)).to_frame().sort_index().T","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:46.525331Z","iopub.execute_input":"2022-04-14T19:15:46.525693Z","iopub.status.idle":"2022-04-14T19:15:46.564055Z","shell.execute_reply.started":"2022-04-14T19:15:46.525664Z","shell.execute_reply":"2022-04-14T19:15:46.563441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Images","metadata":{}},{"cell_type":"code","source":"train_path = \"../input/histopathologic-cancer-detection/train\"\n\nsample = train.sample(n=16).reset_index()\n\nplt.figure(figsize=(6,6))\n\nfor i, row in sample.iterrows():\n\n    img = mpimg.imread(f'../input/histopathologic-cancer-detection/train/{row.id}')    \n    label = row.label\n\n    plt.subplot(4,4,i+1)\n    plt.imshow(img)\n    plt.text(0, -5, f'Class {label}', color='k')\n        \n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:46.9117Z","iopub.execute_input":"2022-04-14T19:15:46.911958Z","iopub.status.idle":"2022-04-14T19:15:47.664617Z","shell.execute_reply.started":"2022-04-14T19:15:46.91193Z","shell.execute_reply":"2022-04-14T19:15:47.663984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Validation Sets","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = train_test_split(train, test_size=0.2, random_state=1, stratify=train.label)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:47.665787Z","iopub.execute_input":"2022-04-14T19:15:47.667581Z","iopub.status.idle":"2022-04-14T19:15:47.996325Z","shell.execute_reply.started":"2022-04-14T19:15:47.66754Z","shell.execute_reply":"2022-04-14T19:15:47.99554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Generators","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1/255)\nvalidation_datagen = ImageDataGenerator(rescale=1/255)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:47.998127Z","iopub.execute_input":"2022-04-14T19:15:47.998428Z","iopub.status.idle":"2022-04-14T19:15:48.003554Z","shell.execute_reply.started":"2022-04-14T19:15:47.998375Z","shell.execute_reply":"2022-04-14T19:15:48.002572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)\n\nvalid_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:15:48.061172Z","iopub.execute_input":"2022-04-14T19:15:48.061388Z","iopub.status.idle":"2022-04-14T19:16:57.011993Z","shell.execute_reply.started":"2022-04-14T19:15:48.061364Z","shell.execute_reply":"2022-04-14T19:16:57.011237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TR_STEPS = len(train_loader)\nVA_STEPS = len(valid_loader)\n\nprint(TR_STEPS)\nprint(VA_STEPS)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:16:57.013945Z","iopub.execute_input":"2022-04-14T19:16:57.014381Z","iopub.status.idle":"2022-04-14T19:16:57.021772Z","shell.execute_reply.started":"2022-04-14T19:16:57.014345Z","shell.execute_reply":"2022-04-14T19:16:57.021066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Base Model","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.applications.VGG16(\n    input_shape=(96,96,3), \n    include_top=False, \n    weights='imagenet'\n)\n\nbase_model.trainable = False\n\nbase_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:16:57.022901Z","iopub.execute_input":"2022-04-14T19:16:57.023234Z","iopub.status.idle":"2022-04-14T19:17:00.699077Z","shell.execute_reply.started":"2022-04-14T19:16:57.023198Z","shell.execute_reply":"2022-04-14T19:17:00.697964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build and Train","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)\n\ncnn = Sequential([\n    base_model,\n    BatchNormalization(),\n\n    Flatten(),\n    \n    Dense(16, activation='relu'),\n    Dropout(0.5),\n    Dense(8, activation='relu'),\n    Dropout(0.5),\n    BatchNormalization(),\n    Dense(2, activation='softmax')\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:17:00.702319Z","iopub.execute_input":"2022-04-14T19:17:00.702672Z","iopub.status.idle":"2022-04-14T19:17:00.988729Z","shell.execute_reply.started":"2022-04-14T19:17:00.702641Z","shell.execute_reply":"2022-04-14T19:17:00.988057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = tf.keras.optimizers.Adam(0.001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:17:00.989892Z","iopub.execute_input":"2022-04-14T19:17:00.99012Z","iopub.status.idle":"2022-04-14T19:17:01.008018Z","shell.execute_reply.started":"2022-04-14T19:17:00.990088Z","shell.execute_reply":"2022-04-14T19:17:01.007389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh1 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 40,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T19:17:01.009158Z","iopub.execute_input":"2022-04-14T19:17:01.009375Z","iopub.status.idle":"2022-04-14T20:09:40.750779Z","shell.execute_reply.started":"2022-04-14T19:17:01.009345Z","shell.execute_reply":"2022-04-14T20:09:40.750082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1])\nvis_training(history)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.752082Z","iopub.execute_input":"2022-04-14T20:09:40.752515Z","iopub.status.idle":"2022-04-14T20:09:40.775518Z","shell.execute_reply.started":"2022-04-14T20:09:40.752476Z","shell.execute_reply":"2022-04-14T20:09:40.772984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine Tuning","metadata":{}},{"cell_type":"code","source":"base_model.trainable = True\nk.set_value(cnn.optimizer.learning_rate, 0.00001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.776687Z","iopub.status.idle":"2022-04-14T20:09:40.7771Z","shell.execute_reply.started":"2022-04-14T20:09:40.776875Z","shell.execute_reply":"2022-04-14T20:09:40.776896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.778237Z","iopub.status.idle":"2022-04-14T20:09:40.779161Z","shell.execute_reply.started":"2022-04-14T20:09:40.778917Z","shell.execute_reply":"2022-04-14T20:09:40.778942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train 2  ","metadata":{}},{"cell_type":"code","source":"%%time \n\nh2 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.782282Z","iopub.status.idle":"2022-04-14T20:09:40.782693Z","shell.execute_reply.started":"2022-04-14T20:09:40.782478Z","shell.execute_reply":"2022-04-14T20:09:40.7825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h2.history['auc'] = h2.history['auc_1']\nh2.history['val_auc'] = h2.history['val_auc_1']","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.784079Z","iopub.status.idle":"2022-04-14T20:09:40.78451Z","shell.execute_reply.started":"2022-04-14T20:09:40.784274Z","shell.execute_reply":"2022-04-14T20:09:40.784295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1, h2])\nvis_training(history, start=10)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.785764Z","iopub.status.idle":"2022-04-14T20:09:40.786698Z","shell.execute_reply.started":"2022-04-14T20:09:40.786457Z","shell.execute_reply":"2022-04-14T20:09:40.786482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training 3","metadata":{}},{"cell_type":"code","source":"%%time \n\nh3 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 20,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.793575Z","iopub.status.idle":"2022-04-14T20:09:40.793966Z","shell.execute_reply.started":"2022-04-14T20:09:40.793753Z","shell.execute_reply":"2022-04-14T20:09:40.793775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h3.history['auc'] = h3.history['auc_1'] \nh3.history['val_auc'] = h3.history['val_auc_1'] ","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.795133Z","iopub.status.idle":"2022-04-14T20:09:40.79589Z","shell.execute_reply.started":"2022-04-14T20:09:40.795654Z","shell.execute_reply":"2022-04-14T20:09:40.795678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1, h2, h3])\nvis_training(history, start=10)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.797161Z","iopub.status.idle":"2022-04-14T20:09:40.797583Z","shell.execute_reply.started":"2022-04-14T20:09:40.797354Z","shell.execute_reply":"2022-04-14T20:09:40.797376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.save('HCDv01.h5')\npickle.dump(history, open(f'HCDv01.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.798961Z","iopub.status.idle":"2022-04-14T20:09:40.799642Z","shell.execute_reply.started":"2022-04-14T20:09:40.799396Z","shell.execute_reply":"2022-04-14T20:09:40.799435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\n\nprint('Test Set Size:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.818062Z","iopub.status.idle":"2022-04-14T20:09:40.818483Z","shell.execute_reply.started":"2022-04-14T20:09:40.818252Z","shell.execute_reply":"2022-04-14T20:09:40.818274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['filename'] = test.id + '.tif'","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.819834Z","iopub.status.idle":"2022-04-14T20:09:40.820234Z","shell.execute_reply.started":"2022-04-14T20:09:40.820012Z","shell.execute_reply":"2022-04-14T20:09:40.820033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.821511Z","iopub.status.idle":"2022-04-14T20:09:40.822053Z","shell.execute_reply.started":"2022-04-14T20:09:40.821825Z","shell.execute_reply":"2022-04-14T20:09:40.82185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"../input/histopathologic-cancer-detection/test\"\nprint('Test Images:', len(os.listdir(test_path)))","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.823097Z","iopub.status.idle":"2022-04-14T20:09:40.82365Z","shell.execute_reply.started":"2022-04-14T20:09:40.823421Z","shell.execute_reply":"2022-04-14T20:09:40.823447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ntest_datagen = ImageDataGenerator(rescale=1/255)\n\ntest_loader = test_datagen.flow_from_dataframe(\n    dataframe = test,\n    directory = test_path,\n    x_col = 'filename',\n    batch_size = BATCH_SIZE,\n    shuffle = False,\n    class_mode = None,\n    target_size = (96,96)\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.824719Z","iopub.status.idle":"2022-04-14T20:09:40.825264Z","shell.execute_reply.started":"2022-04-14T20:09:40.825026Z","shell.execute_reply":"2022-04-14T20:09:40.825051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probs = cnn.predict(test_loader)\nprint(test_probs.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.826299Z","iopub.status.idle":"2022-04-14T20:09:40.826846Z","shell.execute_reply.started":"2022-04-14T20:09:40.826616Z","shell.execute_reply":"2022-04-14T20:09:40.826641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(test_loader))\n","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.827953Z","iopub.status.idle":"2022-04-14T20:09:40.828511Z","shell.execute_reply.started":"2022-04-14T20:09:40.828265Z","shell.execute_reply":"2022-04-14T20:09:40.82829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_probs[:10,].round(2))\n","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.829558Z","iopub.status.idle":"2022-04-14T20:09:40.830085Z","shell.execute_reply.started":"2022-04-14T20:09:40.829856Z","shell.execute_reply":"2022-04-14T20:09:40.829882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.argmax(test_probs, axis=1)\nprint(test_pred[:10])","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.831129Z","iopub.status.idle":"2022-04-14T20:09:40.831677Z","shell.execute_reply.started":"2022-04-14T20:09:40.831449Z","shell.execute_reply":"2022-04-14T20:09:40.831475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.832758Z","iopub.status.idle":"2022-04-14T20:09:40.833303Z","shell.execute_reply.started":"2022-04-14T20:09:40.833066Z","shell.execute_reply":"2022-04-14T20:09:40.833092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.label = test_probs[:,1]\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.834343Z","iopub.status.idle":"2022-04-14T20:09:40.834903Z","shell.execute_reply.started":"2022-04-14T20:09:40.834662Z","shell.execute_reply":"2022-04-14T20:09:40.834687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', header=True, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T20:09:40.835973Z","iopub.status.idle":"2022-04-14T20:09:40.836561Z","shell.execute_reply.started":"2022-04-14T20:09:40.836318Z","shell.execute_reply":"2022-04-14T20:09:40.836344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}