{"cells":[{"metadata":{"_uuid":"fe57324f4745fb9d4383198da02a202031625fbb"},"cell_type":"markdown","source":"## Classifying diabetic retinopathy"},{"metadata":{"_uuid":"acec03a5349f652365134989c150cbdf28ba2076","trusted":true},"cell_type":"code","source":"# Put these at the top of every notebook, to get automatic reloading and inline plotting\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14fa222ea053d567aaeb95f9786449c34bf87617","trusted":true},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.transforms import *\nfrom fastai.conv_learner import *\n\nfrom fastai.dataset import *\nfrom fastai.sgdr import *\nfrom fastai.plots import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"276daac1c68af55ffc03e89d20c853bdf23e8522","trusted":true},"cell_type":"code","source":"PATH = \"../input/diabetic-retinopathy-detection/\"\nTMP_PATH = \"/tmp/tmp\"\nMODEL_PATH = \"/tmp/model/\"\nsz=224  # default\n\narch=resnet34\n# sz = 64  # Because medical images.\n# Not sure anymore about sz=64\nbs = 4  # Because we have a very limited sample\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ba310c51d959cab508081d01d87a3aaa2ab5085","trusted":true},"cell_type":"code","source":"print('Make sure cuda is installed:', torch.cuda.is_available())\nprint('Make sure cudnn is enabled:', torch.backends.cudnn.enabled)","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"e7371d1918f4903352b82bb8f4b2ca55fa4386ca"},"cell_type":"markdown","source":"## First look at DR pictures"},{"metadata":{"trusted":true,"_uuid":"00c49e5fe278928d414f0c5310420108468fb40d"},"cell_type":"code","source":"base_image_dir = os.path.join('..', 'input', 'diabetic-retinopathy-detection')\nretina_df = pd.read_csv(os.path.join(base_image_dir, 'trainLabels.csv'))\nretina_df['PatientId'] = retina_df['image'].map(lambda x: x.split('_')[0])\nretina_df['path'] = retina_df['image'].map(lambda x: os.path.join(base_image_dir,\n                                                         '{}.jpeg'.format(x)))\nretina_df['exists'] = retina_df['path'].map(os.path.exists)\nprint(retina_df['exists'].sum(), 'images found of', retina_df.shape[0], 'total')\nretina_df['eye'] = retina_df['image'].map(lambda x: 1 if x.split('_')[-1]=='left' else 0)\n# from keras.utils.np_utils import to_categorical\n# retina_df['level_cat'] = retina_df['level'].map(lambda x: to_categorical(x, 1+retina_df['level'].max()))\n\nretina_df.dropna(inplace = True)\nretina_df = retina_df[retina_df['exists']]\nretina_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"872351c7414530a619335698a7f99c76dcc78293"},"cell_type":"markdown","source":"# Examine the distribution of eye and severity"},{"metadata":{"trusted":true,"_uuid":"a1b9fa58faa7434b0ae7763aaa25d51114552f39"},"cell_type":"code","source":"retina_df[['level', 'eye']].hist(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4165410808218764bcefba7c555de7ec90d01b6"},"cell_type":"markdown","source":"# Check number of images in each classes"},{"metadata":{"trusted":true,"_uuid":"d02a96b2f19df24f66f9a7f6874b89172c701bf0"},"cell_type":"code","source":"retina_df = retina_df[['PatientId', 'level', 'eye', 'path']].drop_duplicates()  # Should not drop any rows in this case\n# V1:\n# retina_df[['level', 'PatientId']].groupby(['level']).agg(['count'])\n\n# V2:\nretina_df.pivot_table(index='level', aggfunc=len).sort_values('PatientId', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a58c730dc6182ff2b0702852c2e1b3af28538a18"},"cell_type":"markdown","source":"# Optional: only keep images of type 0 and 2 (2 being the second most present class in this sample)"},{"metadata":{"trusted":true,"_uuid":"ff9bfae314602bc0bb220c1885a908fdb8496dd3"},"cell_type":"code","source":"# retina_df = retina_df.drop(retina_df[[(x in [1, 3, 4]) for x in retina_df.level]].index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"676a32b80dcacfac4c79aaa015a4f0bc8b4673b2"},"cell_type":"markdown","source":"# Balance the distribution based on the smallest set"},{"metadata":{"trusted":true,"_uuid":"49f8f5b90c0f835b8c6b8d8259e28404db736f1a"},"cell_type":"code","source":"def balance_data(class_size):\n    train_df = retina_df.groupby(['level']).apply(lambda x: x.sample(class_size, replace = True)).reset_index(drop = True)\n    print('New Data Size:', train_df.shape[0], 'Old Size:', retina_df.shape[0])\n    train_df[['level', 'eye']].hist(figsize = (10, 5))\n    return train_df\n\ntrain_df = balance_data(148)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"a9fb6a8048fa18c65800b1865876bf8cccfa0e86","trusted":true},"cell_type":"code","source":"fnames = train_df['path'].values\nlabels = train_df['level'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"284540505be787388d381f9cc8b83e79cdffd2cb"},"cell_type":"markdown","source":"# Making sure fnames and labels are in order"},{"metadata":{"hidden":true,"_uuid":"09f87570c2cd82eb3eed603565240021a5a888fe","trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"test_label = train_df.level.unique()[-1]\n\n# To shuffle rows:\n# train_df = train_df.sample(frac=1).reset_index(drop=True)\n\npatient_example = train_df.loc[train_df['level'] == test_label].iloc[0]\npatient_example_index = train_df.index[train_df['PatientId'] == patient_example['PatientId']][-1]\nprint(patient_example)\nassert labels[patient_example_index] == test_label, f\"Check that patient with id {patient_example_index}'s label is equal to {test_label}\"\n\nimg = plt.imread(f'{fnames[patient_example_index]}')\nplt.imshow(img);","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"5ba79cbbb1fbab9dbcbc35834f62a8b024942d2e","trusted":true},"cell_type":"code","source":"img.shape","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"500f86792c2bd9c9373d883eb301b74219a9b2a7"},"cell_type":"markdown","source":"## Exploring our dataset images size"},{"metadata":{"_uuid":"7aa60a8f5fecaf1e2054fbb53029fa881108ea24","trusted":true,"collapsed":true},"cell_type":"code","source":"data = ImageClassifierData.from_names_and_array(\n    path='./', \n    fnames=fnames, \n    y=labels, \n    classes=sorted(retina_df.level.unique()), \n    test_name=None, \n    tfms=tfms_from_model(arch, sz)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9de7795ab29147f13b56eee41b68e9deeebf76a3","collapsed":true},"cell_type":"code","source":"img_name = data.trn_ds.fnames[0]; img_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18cef0b27ecae456d0dcce3ed3438c9e7ded20ed","collapsed":true},"cell_type":"code","source":"img = PIL.Image.open(img_name); img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1490c7a4f401486701f720570df3db9d72e50e9d","collapsed":true},"cell_type":"code","source":"img.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3db47529f7f4605de105fa85c5e59a910e695c79","collapsed":true},"cell_type":"code","source":"size_d = {k: PIL.Image.open(k).size for k in data.trn_ds.fnames}\nrow_sz, col_sz = list(zip(*size_d.values()))\nrow_sz = np.array(row_sz); col_sz = np.array(col_sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9818d84119b66c1df601fd9eb538a53ed7f86dee","collapsed":true},"cell_type":"code","source":"plt.hist(row_sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e613ea7fd4d1bdc1de7ace5a8ff25fb3b2613ea1","collapsed":true},"cell_type":"code","source":"plt.hist(col_sz[col_sz < 2000])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13737519db896adac6836afa9048c55b0e79ef02"},"cell_type":"markdown","source":"We can't really balance the size of our dataset by down-sampling because almost all images are very large, because of this we are going to resize our images instead."},{"metadata":{"trusted":true,"_uuid":"0b432a2fc1832a8dbf74a0bdc2c9fcfbbbd847c3"},"cell_type":"code","source":"def get_data(sz, bs=4): # sz: image size, bs: batch size\n#     tfms = tfms_from_model(arch, sz, aug_tfms=transforms_top_down, max_zoom=1.05)\n    tfms = tfms_from_model(arch, sz)\n    data = ImageClassifierData.from_names_and_array(\n        path='./', \n        fnames=fnames, \n        y=labels, \n        classes=sorted(retina_df.level.unique()), \n        test_name=None,\n        tfms=tfms,\n        bs=bs\n    )\n    \n    if len(data.trn_ds) % bs == 1:\n        data = ImageClassifierData.from_names_and_array(path='./', classes=sorted(retina_df.level.unique()), test_name=None, tfms=tfms, bs=bs,\n            fnames=fnames[:-1], \n            y=labels[:-1]\n        )\n    assert len(data.trn_ds) % bs != 1, 'This condition makes sure that we never have a batch size of 1, which could cause issues with lr_find for instance.'\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7f235a3d83dfd231e319b8bd57b3b5efc6a0252"},"cell_type":"code","source":"data = get_data(sz=sz, bs=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a29fc7716fac9adbbefbb4d9cc8fe24e46b85189"},"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\nfrom fastai.metrics import accuracy, recall, precision, fbeta\n\nquadratic_kappa = lambda y_hat, y: cohen_kappa_score(y_hat, y, weights='quadratic')\ndef f2(log_preds, targs): \n    return fbeta(log_preds, targs, 2)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"scrolled":false,"_uuid":"34c2f123f94ac01f68f8ab36a2943486fbe35ab5","trusted":true},"cell_type":"code","source":"print(f'Sample classes: {retina_df.level.unique()}')\n\nlearn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f82774182c194efe390365113ea46c3a643c2d40"},"cell_type":"code","source":"learn.fit(1e-2, 3, metrics=[accuracy, recall, precision, f2])\n# learn.fit(1e-2, 3, metrics=[quadratic_kappa])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c06c11b0c87e15296ae71ebc1aaf7a94ae56e8d"},"cell_type":"code","source":"def get_80percent_accuracy_with_sample_bias():\n    fnames2 = retina_df['path'].as_matrix()[:-1]\n    labels2 = retina_df['level'].as_matrix()[:-1]\n\n    data = ImageClassifierData.from_names_and_array(\n        path='./',\n        fnames=fnames2,\n        y=labels2,\n        classes=sorted(retina_df.level.unique()),\n        test_name=None,\n        tfms=tfms_from_model(arch, sz, aug_tfms=transforms_top_down, max_zoom=1.05)\n    )\n\n    print(retina_df.pivot_table(index='level', aggfunc=len).sort_values('PatientId', ascending=False))\n\n    return ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)\n\nlearn = get_80percent_accuracy_with_sample_bias()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a0bfc82e06daef79e4f339dceaf16ee0f1b92c1"},"cell_type":"code","source":"learn.fit(0.01, 2, metrics=[accuracy, recall, precision, f2])","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"17f0e5c8170bbcd4a0371c95d5a136e9674f7c31"},"cell_type":"markdown","source":"## Analyzing results: looking at pictures"},{"metadata":{"hidden":true,"_uuid":"b986461621fef09d1aadfe6c7c04ff21f18eca6e"},"cell_type":"markdown","source":"As well as looking at the overall metrics, it's also a good idea to look at examples of each of:\n1. A few correct labels at random\n2. A few incorrect labels at random\n3. The most correct labels of each class (i.e. those with highest probability that are correct)\n4. The most incorrect labels of each class (i.e. those with highest probability that are incorrect)\n5. The most uncertain labels (i.e. those with probability closest to 0.5)."},{"metadata":{"hidden":true,"_uuid":"942e14027848dcad6d6c80422b5bbf56f63bc1c7","trusted":true,"collapsed":true},"cell_type":"code","source":"# This is the label for a val data\ndata.val_y","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"6378a93c7e03118e0b12d423d31b2be8f95c1005","trusted":true,"collapsed":true},"cell_type":"code","source":"# from here we know that 'cats' is label 0 and 'dogs' is label 1.\ndata.classes","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"1326d53d70acad37272afa71c20d6ff39e165c89","trusted":true,"collapsed":true},"cell_type":"code","source":"# this gives prediction for validation set. Predictions are in log scale\nlog_preds = learn.predict()\nlog_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"b7ff16accb5fa5104de318bd261abcdee72530aa","trusted":true,"collapsed":true},"cell_type":"code","source":"log_preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"b8709f3bac05d50ee2f7b0b774eb86af8f0f709c","trusted":true},"cell_type":"code","source":"preds = np.argmax(log_preds, axis=1)  # from log probabilities to 0 or 1\nprobs = np.exp(log_preds[:,1])        # pr(dog)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"37901ee29c7a90714b6385d51eb4126b890f4f8e","trusted":true},"cell_type":"code","source":"def rand_by_mask(mask): return np.random.choice(np.where(mask)[0], 4, replace=False)\ndef rand_by_correct(is_correct): return rand_by_mask((preds == data.val_y)==is_correct)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"f9de40f3a17b608fb5c4baea893e138c88464fd7","trusted":true},"cell_type":"code","source":"def plots(ims, figsize=(12,6), rows=1, titles=None):\n    f = plt.figure(figsize=figsize)\n    for i in range(len(ims)):\n        sp = f.add_subplot(rows, len(ims)//rows, i+1)\n        sp.axis('Off')\n        if titles is not None: sp.set_title(titles[i], fontsize=16)\n        plt.imshow(ims[i])","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"f64c7bef125f50701884273ed8d161de25b9703b","trusted":true},"cell_type":"code","source":"def load_img_id(ds, idx): return np.array(PIL.Image.open(ds.fnames[idx]))\n\ndef plot_val_with_title(idxs, title):\n    imgs = [load_img_id(data.val_ds,x) for x in idxs]\n    title_probs = [probs[x] for x in idxs]\n    print(title)\n    return plots(imgs, rows=1, titles=title_probs, figsize=(16,8))","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"9b2fac071b6f72c82cb4a499dffe9dc77b5b57e3","trusted":true,"collapsed":true},"cell_type":"code","source":"# 1. A few correct labels at random\nplot_val_with_title(rand_by_correct(True), \"Correctly classified\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"2380c1cc7a75c19c96a72fdafed058c7fbd92815","trusted":true,"collapsed":true},"cell_type":"code","source":"# 2. A few incorrect labels at random\nplot_val_with_title(rand_by_correct(False), \"Incorrectly classified\")","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"c0782f2afa83cc74dac7239cb7d6fa5d7f034aab","trusted":true},"cell_type":"code","source":"def most_by_mask(mask, mult):\n    idxs = np.where(mask)[0]\n    return idxs[np.argsort(mult * probs[idxs])[:4]]\n\ndef most_by_correct(y, is_correct): \n    mult = -1 if (y==1)==is_correct else 1\n    return most_by_mask(((preds == data.val_y)==is_correct) & (data.val_y == y), mult)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"127110ef2fedc2f795b08d494adf9ce7d666d88c","trusted":true,"collapsed":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(0, True), \"Most correct 0\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"c59754e3ca49c83553d7b86959071fd331a3e4f2","trusted":true,"collapsed":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(2, True), \"Most correct 2\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"e47ad9fca54f184cf737e77103987ea6d89ebddc","trusted":true,"collapsed":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(0, False), \"Most incorrect 0\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"5f4b6c4800ce0c32a3619205607e574231b5bae1","trusted":true,"collapsed":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(2, False), \"Most incorrect 2\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"3fa2abbbf44c4b364d8415df7566419742935146","trusted":true,"collapsed":true},"cell_type":"code","source":"most_uncertain = np.argsort(np.abs(probs -0.5))[:4]\nplot_val_with_title(most_uncertain, \"Most uncertain predictions\")","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"9881009afc9dae99a579b5d7409dbc662a61b638"},"cell_type":"markdown","source":"## Choosing a learning rate"},{"metadata":{"hidden":true,"_uuid":"14336cf080fda4f6c9ef0f6b1d59367353e3129d"},"cell_type":"markdown","source":"The *learning rate* determines how quickly or how slowly you want to update the *weights* (or *parameters*). Learning rate is one of the most difficult parameters to set, because it significantly affects model performance.\n\nThe method `learn.lr_find()` helps you find an optimal learning rate. It uses the technique developed in the 2015 paper [Cyclical Learning Rates for Training Neural Networks](http://arxiv.org/abs/1506.01186), where we simply keep increasing the learning rate from a very small value, until the loss stops decreasing. We can plot the learning rate across batches to see what this looks like.\n\nWe first create a new learner, since we want to know how to set the learning rate for a new (untrained) model."},{"metadata":{"trusted":true,"_uuid":"ed355f0d5228976496a380f7a136bb6ad3bff7fd"},"cell_type":"code","source":"data = get_data(sz=224, bs=4)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"5564b4234690421173ae39c0838d84ad119b0d16","trusted":true},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"scrolled":true,"_uuid":"9c6b0f525504ab510c3b7eb93f86abf57c85780c","trusted":true},"cell_type":"code","source":"lrf=learn.lr_find(start_lr=1e-7, end_lr=1e-1)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"4d12db6775555887d09fa4fdbfbef0395c28b866"},"cell_type":"markdown","source":"We can see the plot of loss versus learning rate to see where our loss stops decreasing:"},{"metadata":{"hidden":true,"_uuid":"8b482b17a6580590ddf2014561fc9052b148637a","trusted":true},"cell_type":"code","source":"learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"701680e992c45ee0cf3e273aef1a9cc2232cf0ec"},"cell_type":"markdown","source":"The loss is still clearly improving at lr=1e-2 (0.01), so that's what we use. Note that the optimal learning rate can change as we train the model, so you may want to re-run this function from time to time."},{"metadata":{"_uuid":"ac3b6fd2aed9cab23bfd01482db48a8fab8eb337"},"cell_type":"markdown","source":"## Improving our model"},{"metadata":{"trusted":true,"_uuid":"18d9dfcfd270466d5d333a811cede42e18cefca5"},"cell_type":"code","source":"lr = 3e-3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9181a07ef4eb01a3521b8c72ad8718604af61224"},"cell_type":"code","source":"learn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b8262fc451785115cbb5fe53c626ab221090e55"},"cell_type":"code","source":"learn.fit(lr, 3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd395d92b1531da3de0d96f9456ae168867f4f94"},"cell_type":"code","source":"lrs = np.array([lr/9,lr/3,lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea2faf85be262f2779e40a63769098c08b0917e1"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit(lr, 3, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"5ebcae9477b240a02026779ef51f2b1e591719ac"},"cell_type":"markdown","source":"By default when we create a learner, it sets all but the last layer to *frozen*. That means that it's still only updating the weights in the last layer when we call `fit`."},{"metadata":{"hidden":true,"_uuid":"346f9a8c8b4b05e56941ea2f6cac954f3f84c198"},"cell_type":"markdown","source":"What is that `cycle_len` parameter? What we've done here is used a technique called *stochastic gradient descent with restarts (SGDR)*, a variant of *learning rate annealing*, which gradually decreases the learning rate as training progresses. This is helpful because as we get closer to the optimal weights, we want to take smaller steps."},{"metadata":{"hidden":true,"_uuid":"56de6b06af18a856a076de6d1eed1945b268a07a","trusted":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"f9f0745a19a6ade9de4588ee65c2a7065fc40ed9"},"cell_type":"markdown","source":"Our validation loss isn't improving much, so there's probably no point further training the last layer on its own."},{"metadata":{"hidden":true,"_uuid":"7e3fc80daed27e97c2299a0bf8d4cad409e5c788"},"cell_type":"markdown","source":"Since we've got a pretty good model at this point, we might want to save it so we can load it again later without training it from scratch."},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"e8dab6b7777e0a41c50934b155ab6eedab6a76f2","trusted":true},"cell_type":"code","source":"learn.save('224')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"_uuid":"cf96b675d212554de2a045b495d479e2b210cd9a","trusted":true},"cell_type":"code","source":"learn.load('224')","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"cb6102e09fd3c172d9f3ff644b97a6fa488b24c9"},"cell_type":"markdown","source":"There is something else we can do with data augmentation: use it at *inference* time (also known as *test* time). Not surprisingly, this is known as *test time augmentation*, or just *TTA*.\n\nTTA simply makes predictions not just on the images in your validation set, but also makes predictions on a number of randomly augmented versions of them too (by default, it uses the original image along with 4 randomly augmented versions). It then takes the average prediction from these images, and uses that. To use TTA on the validation set, we can use the learner's `TTA()` method."},{"metadata":{"hidden":true,"_uuid":"455b5b3f31f3b3f51773f3d7f3bb7752b9fa28bc","trusted":true},"cell_type":"code","source":"log_preds,y = learn.TTA()\nprobs = np.mean(np.exp(log_preds),0)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"cfdd304ce317bc907af404f30546572da3d2a220","trusted":true},"cell_type":"code","source":"accuracy_np(probs, y)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"fcb92d519aaa2d5a8982a6f7c280ec6ee6a0916d"},"cell_type":"markdown","source":"I generally see about a 10-20% reduction in error on this dataset when using TTA at this point, which is an amazing result for such a quick and easy technique!"},{"metadata":{"_uuid":"3068e3a8473a2c1ac558780c107d0158bd338e51"},"cell_type":"markdown","source":"## Analyzing results"},{"metadata":{"heading_collapsed":true,"_uuid":"667093c0f80a6477a5d25c4fbe3f0e071a9520ab"},"cell_type":"markdown","source":"### Confusion matrix "},{"metadata":{"hidden":true,"_uuid":"d9a8432f73faf42db4d6cd386a07667b6d813b59","trusted":true},"cell_type":"code","source":"preds = np.argmax(probs, axis=1)\nprobs = probs[:,1]","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"5c75497abd825d42fb477f3e869d79c332f8aeba"},"cell_type":"markdown","source":"A common way to analyze the result of a classification model is to use a [confusion matrix](http://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/). Scikit-learn has a convenient function we can use for this purpose:"},{"metadata":{"hidden":true,"_uuid":"8b014d4a3e735cafdb1aa8d53d2dc6847be12367","trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y, preds)\ncm","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"e1efa2ca5de43c3c212ac4ee12e9f9d90eae4598"},"cell_type":"markdown","source":"We can just print out the confusion matrix, or we can show a graphical view (which is mainly useful for dependents with a larger number of categories)."},{"metadata":{"hidden":true,"_uuid":"8b005778bedce93b8c9754d1fed58c98404f553d","trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, data.classes)","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"22ca13a90fe2dda24234ae5bc7d2a712d740d03f"},"cell_type":"markdown","source":"### Looking at pictures again"},{"metadata":{"hidden":true,"_uuid":"2c1c45c78ce3e9fa4b94b5fb0b232142cc8cbc8d","trusted":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(0, False), \"Most incorrect 0\")","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"fbb1f131b8f1abb95223a5850f37991fe918ac27","trusted":true,"collapsed":true},"cell_type":"code","source":"plot_val_with_title(most_by_correct(1, False), \"Most incorrect 1\")","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"3e3db3bd9fe817641063ff0e37bfb4bcf6b33967"},"cell_type":"markdown","source":"## Review: easy steps to train a world-class image classifier"},{"metadata":{"hidden":true,"_uuid":"54eef5b76bc4e37be5aee47b7eab68fe04bea567"},"cell_type":"markdown","source":"1. precompute=True\n1. Use `lr_find()` to find highest learning rate where loss is still clearly improving\n1. Train last layer from precomputed activations for 1-2 epochs\n1. Train last layer with data augmentation (i.e. precompute=False) for 2-3 epochs with cycle_len=1\n1. Unfreeze all layers\n1. Set earlier layers to 3x-10x lower learning rate than next higher layer\n1. Use `lr_find()` again\n1. Train full network with cycle_mult=2 until over-fitting"},{"metadata":{"hidden":true,"_uuid":"51e3570dd17c5f6f4200f71c97eee9660f45b5cc"},"cell_type":"markdown","source":"`ConvLearner.pretrained` builds *learner* that contains a pre-trained model. The last layer of the model needs to be replaced with the layer of the right dimensions. The pretained model was trained for 1000 classes therfore the final layer predicts a vector of 1000 probabilities. The model for cats and dogs needs to output a two dimensional vector. The diagram below shows in an example how this was done in one of the earliest successful CNNs. The layer \"FC8\" here would get replaced with a new layer with 2 outputs.\n\n<img src=\"https://image.slidesharecdn.com/practicaldeeplearning-160329181459/95/practical-deep-learning-16-638.jpg\" width=\"500\">\n[original image](https://image.slidesharecdn.com/practicaldeeplearning-160329181459/95/practical-deep-learning-16-638.jpg)"},{"metadata":{"hidden":true,"_uuid":"0ad6dd4ebc10ad532f13277dc3dd45fa10629309"},"cell_type":"markdown","source":"```python\nlearn = ConvLearner.pretrained(resnet34, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH)\n```\n*Parameters*  are learned by fitting a model to the data. *Hyperparameters* are another kind of parameter, that cannot be directly learned from the regular training process. These parameters express “higher-level” properties of the model such as its complexity or how fast it should learn. Two examples of hyperparameters are the *learning rate* and the *number of epochs*.\n\nDuring iterative training of a neural network, a *batch* or *mini-batch* is a subset of training samples used in one iteration of Stochastic Gradient Descent (SGD). An *epoch* is a single pass through the entire training set which consists of multiple iterations of SGD.\n\nWe can now *fit* the model; that is, use *gradient descent* to find the best parameters for the fully connected layer we added. We need to pass two hyperameters: the *learning rate* (generally 1e-2 or 1e-3 is a good starting point, we'll look more at this next) and the *number of epochs* (you can pass in a higher number and just stop training when you see it's no longer improving, then re-run it with the number of epochs you found works well.)"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"41edb90ac9f71cefeabcce2b4ce321a71ed565d6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c56c12fd3a7285ad8202a61a0d4b708cbb8494cb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e1899f5ffa2d91e0b707dc1287fd94ba9bb0e12a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"65bf52d8a0e1f88d74a591cb52b15057585eb689"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"68bc449f4cf5b151c05d41e378f199dae49893d1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"33201849cbbf6c98b9f579b3ddea2c6668f953d3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"598f1cdf52313b23255a2ed91e1b3b115607e222"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"663cfe29ae2615278727e33699b3992cda1071ce"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2add4f0e6a5753156d7dceb23ffef6b344929de8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9bc12b0c1f6827ff3d261bed12181f922cd8f55f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0ac3137c6a59935554a3ba1479895419a90ec8b3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a6eb8feeb70f9c2c73b17175ed2e8f24c6f020c7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fbd0fca1b1dc0542ea0cbc2864aba3872b9e4a45"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"489d228ac104678933078e56b459334a7415667a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"57eacad4666ffd68722d6f8c7fea78196bd52c13"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2dcd1cf93a8edd1b2f90b3891fe3ff4690eac473"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5090c045aedf6da8d219e7335d72dafaaa2185b2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"90a504a64577599298fb4d3584fd5313525c7a50"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1544ca98e7407e3fdc34a2d6f621215dc0fb12c8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1a4f100e0f6d6200d219e453b29758821463b770"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"249b9e7be9632cce1be73c5814f1d10f548e161c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f21ef44e86a63fd8a9491592c4de6add08341032"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"92ad9475f7bfe4c7440f7c66cb099322602aea25"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f2ac5d46da1c358aae867326721f650bfc948e67"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"98d8597c583c49bad2dd2d01631918bdc366dcdf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0b83f803745b0b77ee31fe1b40bb4a63a7a14968"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"65fa6dbea399b77ed96ec2363690892622452c4e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aed136433e68799af3586e5e62d9f630e486a933"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a9e67518d59859f8cd6258f323567191313448a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"34625b5d59c6334b082304a040df516aebcdb0cb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1a748b06ed651d08de2d0289ce6463d89958e870"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"344ef41836881f6119a4594a6aca67df36e9d0ca"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"edbf33ef58b87fd222b32fe6fae6d205bfb50158"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3b28919de8a4993d909ae6844e9677fa1b331167"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c5c4eabfcc60bf3e54394409b65eea48e5f00004"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"363726bb47ba319e5d752ceb99c8572153cf10ea"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"29bc77744a6caae3ecdc196347d869ecc79fa1b3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f737cad9ddf5e59ee584a834e24378dcc09a2596"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"10e85a28e90cae8aebd2146b4c437155de61d783"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"abf62697e41aca32fd626890461b60932da716db"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c3f914b675eff9cd8fa7db0cd0ed2b269f47baf6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"473fc3081a76fc496e49c2ef7c62b79b3966b26a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"28b368d5c62caea53c75259d210f77151c008654"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d672514feb556015dba86fcb70804676b406fa3d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3bec67c27271dc0083963be7fd04884012f92a40"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8a78034c070bfccbe722b0622b89c7e18541c42d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"198d094f218dbbf9abd464659741c73fa63f5b90"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9f3e7f9f24fe6bdbc6ad477b2ab932a806b6accb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b891f851dedae589ba052e00db14edff57d9b3c0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"775cf2a354deb73783e3f9837b4fedfad0450e88"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2c9590d7018e7b7e7225b751c5b331b033237622"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0cf81b216313ffda6a7efed90113eff9bbaf8bf3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b61b26177186ab793b27c508db0a6dd66e81657e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"93ae0b864e9c955d0b946c1756ee733eca0e1e8c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2e3f718570e9ba71ba10d63aeaa6d510150a0024"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8450644bf2447b8b64eb1c62d454223bd5096c50"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"54c272231f60f6e00a0021c6303396cc02a39c21"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"473d59273ada93523fb4648f9266057d57436f25"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"81c6c0c04e862f0d77a9a66ff9857c32934ba65a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ae7dc264c9627a0f9c0ecce78671ba5919fe0e12"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8074e7f8860f574c139f0cd925e06c259527dfe4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9ae2eee5ca8f61e6795e59c5a98c9237b8dd3f34"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"bfa088d03771692876b72a9cf82830b7f9f61951"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c6d193a9cdcdbede0f771b866bd3a4c5345af2b6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cf3242d38212723b063d908561271e40c0c37a51"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9a87c0fba01373091abd8991dc987b6fe3ef8c6a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cc9bfd5f02031717223fb259e449774b8622f147"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e3c34dcbe3939b0d69de914e23039244276a1aa0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3ee5ce269015f3aee96c78bbf4913d32bb93befa"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f91c49d8d6d2f49540619fc2c1e446e99e088b3d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6fd8a7f9463d80cc5213dba022eb390ed55aeedb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a79fde4d0b6eb628c9cd67bcf8b74c886b8cc9a4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d0d573dd1a822e0efa3e3e055605f505c4630e97"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"45e1294c1a575a16cb082f84b6772f8c7bc97f3b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c6875176c0d109edf5565c3138acd19af7bbf514"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9c3f7f0a20ad833268694a05bb848d479796f1cc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"93693fc950f2c6e5e53dabed0533a9985a4ed75d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f1ef6fcffd76a9b11bebfab41ef6742d3c24ac7b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9d5fac7c97bd7bf9eefe6dffe838a5200e4df5b9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5316408055e706c504764e03f1a4696cb7eff829"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"78d86f456ba9e9bc4abb77ea9ae018bfade86d1d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dbfaaa871b966a8faeed609080eb6d37f6b63b8b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d6b57fce85ee9e0d8f3a6473f44e889af03df9af"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e891d7706af67bc3f40f01a256ad4dabb13c4b2d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"75fa0c9cf18eb681bc6d07ee6bce98904de471c0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1dd441393e1a51226a0a06317a2709024e0a9ebd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"72882310cfbb31ae9bdda786e516af630979ab7e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ccde99e7f28b70a8335ffae8f857262b16959c76"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f3c4d7e209447f9c78e01e7adfb6d0e9939b7fbf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0c605ed34af8757c5e9d5b12bce227cfc15f9072"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dae00c8a8560115c113e50f33386b863bad16695"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fa8c807887f9b035f04158c26a105e23366ae12c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"728f6ad0fed11a6ae6a9018da3797cd4c70bff9a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f9a125ada203b82ff7d9f4b234a3ea2eb71f9a65"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cf05a22323342e1613604a39d0f2dc452316f252"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"94d6ec4577343aecab33f4e201e6cd30adc30a0d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cc99c1e9383894f6d5e6200bfbd436f0897c397f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1abfb679bd6e547882fc42aedd1f660c97a33117"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"78faecc39123fb4b96293ab33f7741608e6449fc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a359e429903489e02e04432eb3e639a9a3a12023"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8eb419eef401f05f5ba12e41d7d7971cf95b9e55"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"bacf383ec7474db366963bfe6ac9d9e5525f018b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2283c12a9731fa67b34c3ae261193711eb319886"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"06d1bb984520a1d7530c41fec97171a3d286c0a2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ef16b082b55002305fdd8bd3a90f13b743330964"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"582994d26c8027086838a43ca50d38f82fa7c881"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"268fc333a3ccda191509ee95482eb7895c7cfd89"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"db9d44ae756b299f2433bb3077496e319ef72152"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ea3eb20d3a49d463e3d64b9a6038eba93f558efd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c567f1d0a1c18aa7f1f68b547efcc0079473f80e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d11bf972cef5f384d2b40a43e4b4b5c29698507b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"23ec6b86f5af476aa4865bd19f9c167ebb37939d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0ceb1dc35d752bc60e05fa28dccc631adf646960"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ce04395c431f8e5bb918a1b7a74a1858e1bab7d9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ba22e96ba84053624d328ce76e5aa90b834a16c9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ae7f3f5ca5207ec8655669e2d84f671959a760b0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"efea5412e78494e4436dc8f13f303a1248e107eb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"213e0ca940786146dd8748420885387d9b1c3ae3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d6a99b4cd87eb7fc66838e4a25c685a51001776e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4caef8cbe61e68ba243f853c87ccd73cd81c3473"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"38b211bb0060eee376a77f13e6931b9dd3f00080"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a8a5f10760a83d448eeac30fa7270f0172197738"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c31b2616fa2f1f042edfa257f19ad405c0478f85"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"13a00defe4f8e82f1f0855fed9c9879d06476190"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c3bb056278108b427c7228ffbff9052d7aab7191"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c596483fe6a4dc78c835510c39f62b284b035e77"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"35e3788f79b29a2c1348af9b5f3494e8a19c5932"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"04705037e198036b6ffbc2640dd2fcbde3ef64f8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b89cfd76ae4d30ae21f3e5c965e7fc58b0d78917"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"eb5aa8bac67ab21baa9768847ffbf9fb48413c5c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"67908067ff1fa0333a9c523c6a731af9229fd6e8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1aad276c5d9f049b5db5b760c631cb44e1b0ef0c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5c7e30fb7fc57d73ee6937f542073c35eb3ce14a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6ada10a434294e843e3bcea445d4781119edcbf2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6a8edd8a9f9957e0c6597ccbb684f7b39844b7a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"881ec75cfffd7e265b8dac06e679a72b46feb546"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"13129bb17b31e57031576a5361d4f6fa6b102087"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a14fa9b34fb5aebdc537ba6e470d8156648d8a82"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ff923c24754c994740d94586f3a4e1a6c8ce26d7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f08a47201a400ba4b8af78b2fd2b2a0ca105ab00"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"58c8a3dffa0193eae70854924b40956e4432364f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f95e656fdf2308766753e274689cbaad79744af5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9ab3178b888ea1b9656c227f4e8543edf645a422"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"14dab9d34f199c530aa71d566646d52f9170905e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6821d0d9750fb36d5e15bcb20c8fc7f8d69cc67c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1c13720e21bc1f012c71cddc0691b527f60e1666"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"44644fb059908487fa330319eb25d8c654d311de"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"005608b2d37a670ff1e27f49a19b9a7f15d0af3c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6179aeaa2a9168712aed35e41b6ca552a1d207b4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ca41b668e0c7616ddc943cac58bd7edc374a98ba"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"864befd2fbd4e0a19087650778eefb1e14ca3dbc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b4ed099356b642e8ae568eaae2c170fa91d2ec6e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8cea28549c0094eb595a4f312f674ae357c61f55"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"13c5ac6f7fc4ca5102695c83329c801500bebb86"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7b420d098b7cee62fbbcb5941da4c7132a655b7f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b0319626bfd2338807ce29d72d42e574e54e53a2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a9b21891e8b764c809ece1ea1b1ff406616a57a8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0bb4e00beffc44bdb17cf79bf9b19670b0413f11"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3f5953a5fada4129a3db980a535d5450a9c41d45"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"028d8b5dcdf5549a903aa96bbbfc3cc418f922b1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"99061d32f91d90710b698b99a16c0d0a4f8c07b5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"83a725aff96ae64c96072e7dc659ef8538e85ff5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"73f9c34e404fdb447912cc46ddbf9059548ad538"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"80bccc1314476c3662ff4754ca3c514944501b44"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b85afe8bdd1f18813a121e50436c8fc92ce2c52b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9a01ed666477302be4a61011c14ef256805b7d42"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d40842a2c3fb1d5a880ee762fe762697e7c6c48f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e42bf5a6cc4a9c4a83993cedcd0c450497f58e89"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fba8bd2a3da33731de357794463934572d026d87"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"12a39a10e5c43778508acf243547248f32e7f837"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c5d43a4385fe26d2bb3fc833a2bdfd83325b2cdf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"35945899f39fae62a8cb7f6ccab1e44f4db1ae4f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a66ffd052ab44d0b80bc23cd5cec853fa720f622"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d0e38ab90b1c765a8525c12cd34f507472f0f589"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4faa6b45e7d8ce0e85d3b91b353ea1520b3882d5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e1b031958869a813d94efa2924b1171c419d7a9b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9f46563ab7e3c42d2fad974b23df7c3b8848970d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"90fd94bd4c2be2f735d396546e2498ee3d1af9cc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"bf01698dcb507ebd67378c37f0f19ceebe845ccc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dd474f54f824bb627188bea5915f5860b0b91e1b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ae4c505e4360b5df56a95288af43435fe04ed1fb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"081fc00687a84c0e3fcbe6414b8104a65832c700"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"16a0b2727b859a2642210032e25c7e53b002dd5f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"92d2f1b8fe3beccc6a7aad62afd028d4adf993cf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dee97f181193c441083aa5d52edefb0cf219c2ce"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1dd018e35dbbd035305a7b53633b5518f7a5ab6c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"28409b28df929f1727bee9c7bc10e23c9e2aa67e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a58dc4e99ad03063dae6766b119c3cd8b09081ae"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"125136687a695c20651e2b74789fe1547b937cb9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b386cfbddb4f76483958379b6e93248ed629422d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e7e28d277312528bdff9e87f97bb79e7304e2369"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ef50e145a99e7440766f80572725ef8de9ff53f9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d6469d773b483a743ef45454dbef7c4d5b035316"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4f2ec63f83e39af23d87a4f843b7a93e3bdd2b40"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"50e01d4efed129ae6c7bb979fc6e9f0128703023"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2f640593619a2bdf04f69bd70a8f33bf910c5cea"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6e70465e24e3ed7231b2a3986c6f6ba63d039f83"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6ae0084a6d3a0bfd116542ae1afef8c54c37a661"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9b1c584bbec78e08192335b38b3aed32fe6d4dc7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9dc3aef89ce1a560dc3fa155d9bc7a6431a44f50"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5342ed6798cde5e88607bfcd5d2f9b0d4422b196"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6473c52a6fd45e540bd953e5f620e7117feb8109"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d450a9e36705217d2d597bef0d1e02c9c3a6889f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7050c83cd6676373d5e207029c0e5f0227b5748e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0179a544b80edd1115106419b8968a6601f0db8b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"12cc44430f3b289987e7acc41e489a1d3508d9a5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1cc9b9daea60f46cefdae17729cdeedca6e4177e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"adc09d78213a997c32997fef35c65a90fb021acc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7859849b70a115067033d001eacf8638f4017608"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"76d3fa9b2433bf9a43b91e6d815568c1440e3356"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c5acad8f5ed89500bdbf2f9b5fe295ee1c69235f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"65cd32a9281455a657c6391a96ed68adecf151d2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e30d229146d3c4a15f8e0462df70fd948736ca77"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1f9347dec3743bdbdbf11cc73cc32814799f2d65"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5e57d80fcafc5837c1e1ce2ef2da805389c3c304"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"59a5a37acb97ae18b88a59a88d810b84b1e11666"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dc31ad688afc40c792e3551a66bcaf20750d0afb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fdfef7e80dc2934288b8ada5afacf43ee75d79d9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a63090f53f825e334f716bacf6767c40c526a58f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a81c1d24cf92a67e56e17639137a30b39a50db12"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a9f5638644d1b02113bd5e3c5629a2d8fea96f8a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cbaa56b490a150b338febf2a607a8571d06eeed0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"87da3aee84ccceabafc494a7c533cf6de1bd5bc6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ba52648a79ba413cac188f62ce000722e0abb53f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6502db04f7f9f0cdb6c5b814ee515e43f071853a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"20dcd6cb6fde15cc200b9282de5d272eee745465"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"feb3286f0bb12e5d614fc5b5b2fecd2509096a7b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ed3dfc55d4a81e63b7582e7b0c3fa4d58554f76d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e245a7b345d823e609c84f173f93f097cf2e5d78"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4d765e0859b896eed086194da020d20a61aa1026"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"432cde92ebf8ae0f93257d3932a71e4dc2cd508a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cc576320a6a64b8cbc3b033bab4c3d77ad7641fb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f446e32fb839f2a5fd3b15724fa3e862253024a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a7c8c21f7760eeffa7cb9846fb6cd3f3458ddc18"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"995dbe07dabaaa3bb34b070bfa2903e3e3189861"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"414db1ea6f9d6e6c5158fb06658c726422c6e6b3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3117a2d5b5e8fbbf8613d6f373854096ec053ada"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f10a17c8743c1810583bcd8116806998c79f4f07"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"colors":{"hover_highlight":"#DAA520","navigate_num":"#000000","navigate_text":"#333333","running_highlight":"#FF0000","selected_highlight":"#FFD700","sidebar_border":"#EEEEEE","wrapper_background":"#FFFFFF"},"moveMenuLeft":true,"nav_menu":{"height":"266px","width":"252px"},"navigate_menu":true,"number_sections":true,"sideBar":true,"threshold":4,"toc_cell":false,"toc_section_display":"block","toc_window_display":false,"widenNotebook":false}},"nbformat":4,"nbformat_minor":1}