{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel based on my [previouse kernel](https://www.kaggle.com/miklgr500/auto-encoder), if you see one then only last part this kernel will be interesting for you."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport math\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom skimage.transform import resize\nfrom tqdm import tqdm_notebook as tqdm\nimport keras\nfrom keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense, Dropout, Activation, Flatten, Input, BatchNormalization, UpSampling2D, Add\nfrom keras.layers import Conv2D, MaxPooling2D, LeakyReLU\nfrom keras import Model\nfrom keras.optimizers import Adam, Nadam\nfrom sklearn.decomposition import PCA, KernelPCA\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, classification_report","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image1(img,tol=5):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_resized_imgs = []\n\nfor image_id in tqdm(train_df['id_code']):\n    path=f\"../input/train_images/{image_id}.png\"\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n\n    img = crop_image1(img)\n    img = cv2.resize(img, (224, 224))\n    img=cv2.addWeighted (img,4, cv2.GaussianBlur(img , (0,0) , 224/10) ,-4 ,128)\n    train_resized_imgs.append(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_resized_imgs = []\n\nfor image_id in tqdm(test_df['id_code']):\n    path=f\"../input/test_images/{image_id}.png\"\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n\n    img = crop_image1(img)\n    img = cv2.resize(img, (224, 224))\n    img=cv2.addWeighted (img,4, cv2.GaussianBlur(img , (0,0) , 224/10) ,-4 ,128)\n    test_resized_imgs.append(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df['diagnosis'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx = len(train_resized_imgs)\ntrain_resized_imgs.extend(test_resized_imgs)\n\ntrain_resized_imgs = np.expand_dims(train_resized_imgs, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataGen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n        rotation_range=30,  # randomly rotate images in the range (degrees, 0 to 180)\n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        shear_range=0.,  # set range for random shear\n        zoom_range=[0.75, 1.25],  # set range for random zoom\n        channel_shift_range=0.05,  # set range for random channel shifts\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n        rescale=1/255.,\n        # set function that will be applied on each input\n        preprocessing_function=None\n    ).flow(np.array(train_resized_imgs), np.array(train_resized_imgs), batch_size=64)\n\ndef generator():\n    for x, _ in dataGen:\n        yield x, x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <center> AutoEncoder"},{"metadata":{},"cell_type":"markdown","source":"![Autoencoder](https://upload.wikimedia.org/wikipedia/commons/2/28/Autoencoder_structure.png)"},{"metadata":{},"cell_type":"markdown","source":"Pipeline released in this kernel is very simple:\n* train encoder and used one for translate image in vector\n* visulize results vectors used PCA decomposition algorithm\n* do some conclusion about train and test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_encoder(shape=(224, 224, 2)):\n    def res_block(x, n_features):\n        _x = x\n        x = BatchNormalization()(x)\n        x = LeakyReLU()(x)\n    \n        x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = Add()([_x, x])\n        return x\n    \n    inp = Input(shape=shape)\n    \n    # 224\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(inp)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 112\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 32)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 56\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 32)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 28\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 14\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)    \n    \n    # 7\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    \n    x = Conv2D(1, kernel_size=(1, 1), strides=(1, 1), padding='same')(x)\n    return Model(inp, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_decoder(shape=(7, 7, 128)):\n    inp = Input(shape=shape)\n\n    x = UpSampling2D((2, 2))(inp)\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(9, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = Conv2D(1, kernel_size=(1, 1), strides=(1, 1), padding='same')(x)\n    return Model(inp, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import *\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = get_encoder((224, 224, 1))\ndecoder = get_decoder((7, 7, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input((224, 224, 1))\ne = encoder(inp)\nd = decoder(e)\nmodel = Model(inp, d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\ndef mask_mse(y_true, y_pred):\n    mask = 1. - 1. / ( 1. + K.exp(-y_true**2))\n    return K.abs(y_true*mask - y_pred*mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=Nadam(lr=2*1e-3, schedule_decay=1e-5), loss='mse')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(generator(), steps_per_epoch=500, epochs=5, callbacks=[\n    CyclicLR(base_lr=8*1e-4, max_lr=6*1e-3, step_size=250, gamma=0.9)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vec = encoder.predict(np.array(train_resized_imgs)/255.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avec = np.array([v.flatten()\n                 for v in vec])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sc = MinMaxScaler()\navec = sc.fit_transform(avec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx = 3662","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca = PCA(n_components=3)\nemb = pca.fit_transform(avec)\nte_emb = emb[train_idx:]\nemb = emb[:train_idx]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Explained variance ration of PCA"},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.explained_variance_ratio_[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labelMap = {\n    0:'No DR',\n    1:'Mild',\n    2:'Moderate',\n    3:'Severe',\n    4:'Proliferative DR'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\n\nfor t in list(set(y)):\n    plt.plot(emb[np.array(y) == t, 0], emb[np.array(y) == t, 1], '.', label=labelMap[t], alpha=0.75)\nplt.plot(te_emb[:, 0], te_emb[:, 1], '.', label='test data', color='gray', alpha=0.45)\n\nplt.xlabel('component 0')\nplt.ylabel('component 1')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.graph_objs as go\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"init_notebook_mode(connected=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = [go.Scatter3d(\n    x=emb[np.array(y) == t, 0],\n    y=emb[np.array(y) == t, 1],\n    z=emb[np.array(y) == t, 2],\n    mode='markers',\n    marker=dict(\n        size=3,\n        opacity=0.75\n    ),\n    name=labelMap[t]\n)\n    for t in list(set(y))\n]\n\ndata.append(go.Scatter3d(\n    x=te_emb[:, 0],\n    y=te_emb[:, 1],\n    z=te_emb[:, 2],\n    mode='markers',\n    marker=dict(\n        color='#c0c0c0',\n        size=2,\n        opacity=0.75\n    ),\n    name='test data'\n))\n\nlayout = go.Layout(\n    margin=dict(\n        l=0,\n        r=0,\n        b=0,\n        t=0\n    ),\n)\nfig = go.Figure(data=data, layout=layout)\niplot(fig, filename='simple-3d-scatter')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(emb), len(te_emb), len(train_resized_imgs), len(train_df), len(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dist_map = {}\ncount = 0\nfor i, tr in enumerate(emb):\n    dist = np.mean(np.abs(te_emb - tr), axis=-1)\n    dist_map[i] = np.where(dist < 1e-8)[0]\n    if len(dist_map[i]) > 0:\n        for j in dist_map[i]:\n            if count < 10 :\n                fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))\n            \n                ax1.imshow(train_resized_imgs[i, ..., 0])\n                ax1.set_title(f'Train img {i}: {labelMap[y[i]]}')\n            \n                ax2.imshow(train_resized_imgs[3662 + j, ..., 0])\n                ax2.set_title(f'Test img {j}')\n                plt.show()\n            count += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count / len(train_df), count / len(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <center> Conclusion"},{"metadata":{},"cell_type":"markdown","source":"So, about 8% test dataset is leak. \nThus based on PCA decomposition above  and results in the [kernel](https://www.kaggle.com/konradb/adversarial-validation-quick-fast-ai-approach) of Konrad Banachewicz, current  train and test dataset intersection is about 1-3 %. Believe yourself and forget public leaderboard!"},{"metadata":{},"cell_type":"markdown","source":"## <center> Reference\n* https://www.kaggle.com/xhlulu/densenet-keras-starter\n* https://www.kaggle.com/vbookshelf/dr-mobilenet-binary-classifier-tfjs-web-app\n* https://github.com/bckenstler/CLR\n* https://www.kaggle.com/ratthachat/aptos-simple-preprocessing-decoloring-cropping\n* https://www.kaggle.com/konradb/adversarial-validation-quick-fast-ai-approach"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0af50adb4d0c45f9a38d4424fd1c2df9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2bf3967adee741c3ae5d1252068daa23","IPY_MODEL_3a7081b1bb4d4f6eba501f2f874ee486"],"layout":"IPY_MODEL_fbce3fcc81cd4dddb9b3f6457139fc92"}},"24feb04f11e24794bc4ca19657ea3e1b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2bf3967adee741c3ae5d1252068daa23":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_24feb04f11e24794bc4ca19657ea3e1b","max":1928,"min":0,"orientation":"horizontal","style":"IPY_MODEL_9e27147ef1d341eda221ed28ea96c307","value":1928}},"30f32554a27d4269a59227b512b66b60":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"37803fa4119d4a1fbd00def4d6b338a7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_eaf95248c62c46dc89a22510e59d844d","IPY_MODEL_47e6331862df4bd5abf527a868447c84"],"layout":"IPY_MODEL_e19518feda6545508e69215c3e371982"}},"3a7081b1bb4d4f6eba501f2f874ee486":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_60f072a954cc46468661004542fb5fa2","placeholder":"​","style":"IPY_MODEL_ab28b567b8164b28ab23af2d3651b150","value":"100% 1928/1928 [01:50&lt;00:00, 17.38it/s]"}},"47e6331862df4bd5abf527a868447c84":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6b70e2a337ce4e43b6fc0b0df4a8d615","placeholder":"​","style":"IPY_MODEL_80cea31226b5473098d91abd5b7bef39","value":"100% 3662/3662 [07:29&lt;00:00,  7.44it/s]"}},"60f072a954cc46468661004542fb5fa2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"690f7eb8fd164e7e876b67ea9b91400b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6b70e2a337ce4e43b6fc0b0df4a8d615":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"80cea31226b5473098d91abd5b7bef39":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"9e27147ef1d341eda221ed28ea96c307":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ab28b567b8164b28ab23af2d3651b150":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"e19518feda6545508e69215c3e371982":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"eaf95248c62c46dc89a22510e59d844d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_690f7eb8fd164e7e876b67ea9b91400b","max":3662,"min":0,"orientation":"horizontal","style":"IPY_MODEL_30f32554a27d4269a59227b512b66b60","value":3662}},"fbce3fcc81cd4dddb9b3f6457139fc92":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}