{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About\n\nIn this competition, participants are requiered to predict `FVC` and its **_`Confidence`_**.  \nHere, I trained Lightgbm to predict them at the same time by utilizing custom metric.\n\nMost of codes in this notebook are forked from @yasufuminakama 's [lgbm baseline](https://www.kaggle.com/yasufuminakama/osic-lgb-baseline). Thanks!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Library","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport operator\nimport typing as tp\nfrom logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\nfrom functools import partial\n\n\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\n\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfrom PIL import Image\nimport cv2\nimport pydicom\n\nimport torch\n\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_logger(filename='log'):\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nlogger = get_logger()\n\n\ndef seed_everything(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DICT = './'\n\nID = 'Patient_Week'\n# TARGET = 'FVC'\nTARGET = 'virtual_FVC'\nSEED = 42\nVIRTUAL_BASE_FVC = 2000\nseed_everything(seed=SEED)\n\nN_FOLD = 4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading","execution_count":null},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntrain[ID] = train['Patient'].astype(str) + '_' + train['Weeks'].astype(str)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct train input\n\noutput = pd.DataFrame()\ngb = train.groupby('Patient')\ntk0 = tqdm(gb, total=len(gb))\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby('Weeks'):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'}\n        tmp = tmp.drop(columns='Patient_Week').rename(columns=rename_cols)\n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']\n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n    \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make new taret: virutal FVC\ntrain['virtual_FVC'] = train[\"FVC\"] / train[\"base_FVC\"] * VIRTUAL_BASE_FVC","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct test input\n\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\\\n        .rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'})\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\ntest = submission.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient')\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"FVC\"] = np.nan\ntest[\"virtual_FVC\"] = np.nan","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nprint(submission.shape)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train[[ID, 'Patient', TARGET]].copy()\n#Fold = KFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\nFold = GroupKFold(n_splits=N_FOLD)\ngroups = folds['Patient'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nfolds.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom Objective / Metric\n\nThe competition evaluation metric is:\n\n$\n\\displaystyle \\sigma_{clipped} = \\max \\left ( \\sigma, 70 \\right ) \\\\\n\\displaystyle \\Delta = \\min \\left ( \\|FVC_{ture} - FVC_{predicted}\\|, 1000 \\right ) \\\\\n\\displaystyle f_{metric} = - \\frac{\\sqrt{2} \\Delta}{\\sigma_{clipped}} - \\ln \\left( \\sqrt{2} \\sigma_{clipped} \\right) .\n$\n\nThis is too complex to directly optimize by custom metric.\nHere I use negative loglilelihood loss (_NLL_) of gaussian.  \n\nLet $FVC_{ture}$ is $t$ and $FVC_{predicted}$ is $\\mu$, the _NLL_ $l$ is formulated by:\n\n$\n\\displaystyle l\\left( t, \\mu, \\sigma \\right) =\n-\\ln \\left [ \\frac{1}{\\sqrt{2 \\pi} \\sigma} \\exp \\left \\{ - \\frac{\\left(t - \\mu \\right)^2}{2 \\sigma^2} \\right \\} \\right ]\n= \\frac{\\left(t - \\mu \\right)^2}{2 \\sigma^2} + \\ln \\left( \\sqrt{2 \\pi} \\sigma \\right).\n$\n\n`grad` and `hess` are calculated as follows:\n\n$\n\\displaystyle  \\frac{\\partial l}{\\partial \\mu } = -\\frac{t - \\mu}{\\sigma^2} \\ , \\ \\frac{\\partial^2 l}{\\partial \\mu^2 } = \\frac{1}{\\sigma^2}\n$\n\n$\n\\displaystyle \\frac{\\partial l}{\\partial \\sigma}\n=-\\frac{\\left(t - \\mu \\right)^2}{\\sigma^3} + \\frac{1}{\\sigma} = \\frac{1}{\\sigma} \\left\\{ 1 - \\left ( \\frac{t - \\mu}{\\sigma} \\right)^2 \\right \\}\n\\\\\n\\displaystyle \\frac{\\partial^2 l}{\\partial \\sigma^2}\n= -\\frac{1}{\\sigma^2} \\left\\{ 1 - \\left ( \\frac{t - \\mu}{\\sigma} \\right)^2 \\right \\}\n+\\frac{1}{\\sigma} \\frac{2 \\left(t - \\mu \\right)^2 }{\\sigma^3}\n= -\\frac{1}{\\sigma^2} \\left\\{ 1 - 3 \\left ( \\frac{t - \\mu}{\\sigma} \\right)^2 \\right \\}\n$","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"For numerical stability, I replace $\\sigma$ with $\\displaystyle \\tilde{\\sigma} := \\log\\left(1 + \\mathrm{e}^{\\sigma} \\right).$\n\n$\n\\displaystyle l'\\left( t, \\mu, \\sigma \\right)\n= \\frac{\\left(t - \\mu \\right)^2}{2 \\tilde{\\sigma}^2} + \\ln \\left( \\sqrt{2 \\pi} \\tilde{\\sigma} \\right).\n$\n\n$\n\\displaystyle \\frac{\\partial l'}{\\partial \\mu } = -\\frac{t - \\mu}{\\tilde{\\sigma}^2} \\ , \\ \\frac{\\partial^2 l}{\\partial \\mu^2 } = \\frac{1}{\\tilde{\\sigma}^2}\n$\n<br>\n\n$\n\\displaystyle \\frac{\\partial l'}{\\partial \\sigma}\n= \\frac{1}{\\tilde{\\sigma}} \\left\\{ 1 - \\left ( \\frac{t - \\mu}{\\tilde{\\sigma}} \\right)^2 \\right \\} \\frac{\\partial \\tilde{\\sigma}}{\\partial \\sigma}\n\\\\\n\\displaystyle \\frac{\\partial^2 l'}{\\partial \\sigma^2}\n= -\\frac{1}{\\tilde{\\sigma}^2}  \\left\\{ 1 - 3 \\left ( \\frac{t - \\mu}{\\tilde{\\sigma}} \\right)^2 \\right \\}\n\\left( \\frac{\\partial \\tilde{\\sigma}}{\\partial \\sigma} \\right) ^2\n+\\frac{1}{\\tilde{\\sigma}} \\left\\{ 1 - \\left ( \\frac{t - \\mu}{\\tilde{\\sigma}} \\right)^2 \\right \\} \\frac{\\partial^2 \\tilde{\\sigma}}{\\partial \\sigma^2}\n$\n\n, where  \n\n$\n\\displaystyle\n\\frac{\\partial \\tilde{\\sigma}}{\\partial \\sigma} = \\frac{1}{1 + \\mathrm{e}^{-\\sigma}} \\\\\n\\displaystyle\n\\frac{\\partial^2 \\tilde{\\sigma}}{\\partial^2 \\sigma} = \\frac{\\mathrm{e}^{-\\sigma}}{\\left( 1 + \\mathrm{e}^{-\\sigma} \\right)^2}\n= \\frac{\\partial \\tilde{\\sigma}}{\\partial \\sigma} \\left( 1 - \\frac{\\partial \\tilde{\\sigma}}{\\partial \\sigma} \\right)\n$","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class OSICLossForLGBM:\n    \"\"\"\n    Custom Loss for LightGBM.\n    \n    * Objective: return grad & hess of NLL of gaussian\n    * Evaluation: return competition metric\n    \"\"\"\n    \n    def __init__(self, epsilon: float=1e-09) -> None:\n        \"\"\"Initialize.\"\"\"\n        self.name = \"osic_loss\"\n        self.n_class = 2  # FVC & Confidence\n        self.epsilon = epsilon\n    \n    def __call__(self, preds: np.ndarray, labels: np.ndarray, weight: tp.Optional[np.ndarray]=None) -> float:\n        \"\"\"Calc loss.\"\"\"\n        mu = preds[:, 0]\n        sigma = preds[:, 1]\n        sigma_t = np.log(1 + np.exp(sigma))\n        loss_by_sample = ((labels - mu) / sigma_t) ** 2 / 2 + np.log(np.sqrt(2 * np.pi) * sigma_t)\n        loss = np.average(loss_by_sample, weight)\n        \n        return loss\n    \n    def _calc_grad_and_hess(\n        self, preds: np.ndarray, labels: np.ndarray, weight: tp.Optional[np.ndarray]=None\n    ) -> tp.Tuple[np.ndarray]:\n        \"\"\"Calc Grad and Hess\"\"\"\n        mu = preds[:, 0]\n        sigma = preds[:, 1]\n        \n        sigma_t = np.log(1 + np.exp(sigma))\n        grad_sigma_t = 1 / (1 + np.exp(- sigma))\n        hess_sigma_t = grad_sigma_t * (1 - grad_sigma_t)\n        \n        grad = np.zeros_like(preds)\n        hess = np.zeros_like(preds)\n        grad[:, 0] = - (labels - mu) / sigma_t ** 2\n        hess[:, 0] = 1 / sigma_t ** 2\n        \n        tmp = ((labels - mu) / sigma_t) ** 2\n        grad[:, 1] = 1 / sigma_t * (1 - tmp) * grad_sigma_t\n        hess[:, 1] = (\n            - 1 / sigma_t ** 2 * (1 - 3 * tmp) * grad_sigma_t ** 2\n            + 1 / sigma_t * (1 - tmp) * hess_sigma_t\n        )\n        if weight is not None:\n            grad = grad * weight[:, None]\n            hess = hess * weight[:, None]\n        return grad, hess\n    \n    def return_loss(self, preds: np.ndarray, data: lgb.Dataset) -> tp.Tuple[str, float, bool]:\n        \"\"\"Return Loss for lightgbm\"\"\"\n        labels = data.get_label()\n        weight = data.get_weight()\n        n_example = len(labels)\n        \n        # # reshape preds: (n_class * n_example,) => (n_class, n_example) =>  (n_example, n_class)\n        preds = preds.reshape(self.n_class, n_example).T\n        # # calc loss\n        loss = self(preds, labels, weight)\n        \n        return self.name, loss, False\n    \n    def return_grad_and_hess(self, preds: np.ndarray, data: lgb.Dataset) -> tp.Tuple[np.ndarray]:\n        \"\"\"Return Grad and Hess for lightgbm\"\"\"\n        labels = data.get_label()\n        weight = data.get_weight()\n        n_example = len(labels)\n        \n        # # reshape preds: (n_class * n_example,) => (n_class, n_example) =>  (n_example, n_class)\n        preds = preds.reshape(self.n_class, n_example).T\n        # # calc grad and hess.\n        grad, hess =  self._calc_grad_and_hess(preds, labels, weight)\n\n        # # reshape grad, hess: (n_example, n_class) => (n_class, n_example) => (n_class * n_example,) \n        grad = grad.T.reshape(n_example * self.n_class)\n        hess = hess.T.reshape(n_example * self.n_class)\n        \n        return grad, hess\n    \n    \n    def calc_comp_metric(self, preds: np.ndarray, labels: np.ndarray, weight: tp.Optional[np.ndarray]=None) -> float:\n        \"\"\"Calc competition metric.\"\"\"\n        sigma_clip = np.maximum(preds[:, 1], 70)\n        Delta = np.minimum(np.abs(preds[:, 0] - labels), 1000)\n        loss_by_sample = - np.sqrt(2) * Delta / sigma_clip - np.log(np.sqrt(2) * sigma_clip)\n        loss = np.average(loss_by_sample, weight)\n        \n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Utils","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#===========================================================\n# model\n#===========================================================\ndef run_single_lightgbm(\n    model_param, fit_param, train_df, test_df, folds, features, target,\n    fold_num=0, categorical=[], my_loss=None,\n):\n    trn_idx = folds[folds.fold != fold_num].index\n    val_idx = folds[folds.fold == fold_num].index\n    logger.info(f'len(trn_idx) : {len(trn_idx)}')\n    logger.info(f'len(val_idx) : {len(val_idx)}')\n    \n    if categorical == []:\n        trn_data = lgb.Dataset(\n            train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])\n        val_data = lgb.Dataset(\n            train_df.iloc[val_idx][features], label=target.iloc[val_idx])\n    else:\n        trn_data = lgb.Dataset(\n            train_df.iloc[trn_idx][features], label=target.iloc[trn_idx],\n            categorical_feature=categorical)\n        val_data = lgb.Dataset(\n            train_df.iloc[val_idx][features], label=target.iloc[val_idx],\n            categorical_feature=categorical)\n\n    oof = np.zeros((len(train_df), 2))\n    predictions = np.zeros((len(test_df), 2))\n    \n    clf = lgb.train(\n        model_param, trn_data, **fit_param,\n        valid_sets=[trn_data, val_data],\n        fobj=my_loss.return_grad_and_hess,\n        feval=my_loss.return_loss,\n    )\n    oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)\n\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance(importance_type='gain')\n    fold_importance_df[\"fold\"] = fold_num\n\n    predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)\n    \n    # RMSE\n    logger.info(\"fold{} RMSE score: {:<8.5f}\".format(\n        fold_num, np.sqrt(mean_squared_error(target[val_idx], oof[val_idx, 0]))))\n    # Competition Metric\n    logger.info(\"fold{} Metric: {:<8.5f}\".format(\n        fold_num, my_loss(oof[val_idx], target[val_idx])))\n    \n    return oof, predictions, fold_importance_df\n\n\ndef run_kfold_lightgbm(\n    model_param, fit_param, train, test, folds,\n    features, target, n_fold=5, categorical=[], my_loss=None,\n):\n    \n    logger.info(f\"================================= {n_fold}fold lightgbm =================================\")\n    \n    oof = np.zeros((len(train), 2))\n    predictions = np.zeros((len(test), 2))\n    feature_importance_df = pd.DataFrame()\n\n    for fold_ in range(n_fold):\n        print(\"Fold {}\".format(fold_))\n        _oof, _predictions, fold_importance_df =\\\n            run_single_lightgbm(\n                model_param, fit_param, train, test, folds,\n                features, target, fold_num=fold_, categorical=categorical, my_loss=my_loss\n            )\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        oof += _oof\n        predictions += _predictions / n_fold\n\n    # RMSE\n    logger.info(\"CV RMSE score: {:<8.5f}\".format(np.sqrt(mean_squared_error(target, oof[:, 0]))))\n    # Metric\n    logger.info(\"CV Metric: {:<8.5f}\".format(my_loss(oof, target)))\n                \n\n    logger.info(f\"=========================================================================================\")\n    \n    return feature_importance_df, predictions, oof\n\n    \ndef show_feature_importance(feature_importance_df, name):\n    cols = (feature_importance_df[[\"Feature\", \"importance\"]]\n            .groupby(\"Feature\")\n            .mean()\n            .sort_values(by=\"importance\", ascending=False)[:50].index)\n    best_features = feature_importance_df.loc[feature_importance_df.Feature.isin(cols)]\n\n    #plt.figure(figsize=(8, 16))\n    plt.figure(figsize=(6, 4))\n    sns.barplot(x=\"importance\", y=\"Feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('Features importance (averaged/folds)')\n    plt.tight_layout()\n    plt.savefig(OUTPUT_DICT+f'feature_importance_{name}.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## predict \"virutal\" FVC & Confidence(sigma)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train[TARGET]\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [ID, TARGET, 'predict_Week', 'base_Week', \"FVC\", \"base_FVC\"]\nfeatures = [c for c in features if c not in drop_features]\n\nif cat_features:\n    ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')\n    ce_oe.fit(train)\n    train = ce_oe.transform(train)\n    test = ce_oe.transform(test)\n        \nlgb_model_param = {\n    'num_class': 2,\n    # 'objective': 'regression',\n    'metric': 'None',\n    'boosting_type': 'gbdt',\n    'learning_rate': 1e-01,\n    'seed': SEED,\n    'max_depth': 1,\n    \"lambda_l2\": 5e-03,\n    'verbosity': -1,\n}\nlgb_fit_param = {\n    \"num_boost_round\": 10000,\n    \"verbose_eval\":100,\n    \"early_stopping_rounds\": 100,\n}\n\nfeature_importance_df, predictions, oof = run_kfold_lightgbm(\n    lgb_model_param, lgb_fit_param, train, test,\n    folds, features, target,\n    n_fold=N_FOLD, categorical=cat_features, my_loss=OSICLossForLGBM())\n    \nshow_feature_importance(feature_importance_df, TARGET)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert virtual FVC to FVC\noof = oof / VIRTUAL_BASE_FVC * train[[\"base_FVC\"]].values\npredictions = predictions / VIRTUAL_BASE_FVC * test[[\"base_FVC\"]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OSICLossForLGBM().calc_comp_metric(oof, train[\"FVC\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"FVC_pred\"] = oof[:, 0]\ntrain[\"Confidence\"] = oof[:, 1]\ntest[\"FVC_pred\"] = predictions[:, 0]\ntest[\"Confidence\"] = predictions[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission.drop(columns=['FVC', 'Confidence']).merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], \n                                                           on='Patient_Week')\nsub.columns = submission.columns\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}