{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# !pip install xarray==0.16.0\n# !pip download xarray==0.16.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install arviz\n!pip install pymc3==3.8\n\n# !pip download arviz\n# !pip download pymc3==3.8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\n\n\n# import arviz\nimport pymc3 as pm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pm.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"exclude_test_patient_data_from_trainset = True\n\n# train = pd.read_csv('/content/drive/My Drive/Kaggle/OSIC/data_osic/train.csv')\n# df_train = pd.read_csv('/content/drive/My Drive/Kaggle/OSIC/data_osic/train.csv')\ndf_train = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\n\nlatents = pd.read_csv('../input/latent-features/latent_features.csv')\n\nmu_log_var = pd.read_csv('../input/mu-log-varcsv/mu_log_var.csv')\nmu_log_var.columns = ['Patient', \n                      'mu0', 'mu1', 'mu2', 'mu3', 'mu4', \n                      'mu5', 'mu6', 'mu7', 'mu8', 'mu9',\n                      'sig0', 'sig1', 'sig2', 'sig3', 'sig4',\n                      'sig5', 'sig6', 'sig7', 'sig8', 'sig9']\n\ntrain_temp = pd.merge(df_train, latents,on='Patient',how='left')\ntrain = pd.merge(train_temp, mu_log_var, on='Patient', how='left')\n\n# test = pd.read_csv('/content/drive/My Drive/Kaggle/OSIC/data_osic/test.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\n\n# if exclude_test_patient_data_from_trainset:\n#     train = train[~train['Patient'].isin(test['Patient'].unique())]\n\n# train = pd.concat([train, test], axis=0, ignore_index=True)\\\n#     .drop_duplicates()\n\n\n\nle_id = LabelEncoder()\ntrain['PatientID'] = le_id.fit_transform(train['Patient'])\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Male'] = train['Sex'].apply(lambda x: 1 if x == 'Male' else 0)\n\ntrain[\"SmokingStatus\"] = train[\"SmokingStatus\"].astype(\n    pd.CategoricalDtype(['Ex-smoker', 'Never smoked', 'Currently smokes'])\n)\naux = pd.get_dummies(train[\"SmokingStatus\"], prefix='ss')\naux.columns = ['ExSmoker', 'NeverSmoked', 'CurrentlySmokes']\ntrain['ExSmoker'] = aux['ExSmoker']\ntrain['CurrentlySmokes'] = aux['CurrentlySmokes']\n\naux = train[['Patient', 'Weeks', 'Percent']].sort_values(by=['Patient', 'Weeks'])\naux = train.groupby('Patient').head(1)\naux = aux.rename(columns={'Percent': 'Percent_base'})\ntrain = pd.merge(train, aux[['Patient', 'Percent_base']], how='left',\n                 on='Patient')\n\ntrain.head()\n\ntrain['FVC_ref']=round(train['FVC']/(train['Percent']/100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le_id = LabelEncoder()\ntrain['PatientID'] = le_id.fit_transform(train['Patient'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Male'] = train['Sex'].apply(lambda x: 1 if x == 'Male' else 0)\n\ntrain[\"SmokingStatus\"] = train[\"SmokingStatus\"].astype(\n    pd.CategoricalDtype(['Ex-smoker', 'Never smoked', 'Currently smokes'])\n)\naux = pd.get_dummies(train[\"SmokingStatus\"], prefix='ss')\naux.columns = ['ExSmoker', 'NeverSmoked', 'CurrentlySmokes']\ntrain['ExSmoker'] = aux['ExSmoker']\ntrain['CurrentlySmokes'] = aux['CurrentlySmokes']\n\naux = train[['Patient', 'Weeks', 'Percent']].sort_values(by=['Patient', 'Weeks'])\naux = train.groupby('Patient').head(1)\naux = aux.rename(columns={'Percent': 'Percent_base'})\ntrain = pd.merge(train, aux[['Patient', 'Percent_base']], how='left',\n                 on='Patient')\n\ntrain.head()\n\ntrain['FVC_ref']=round(train['FVC']/(train['Percent']/100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_patients = train['Patient'].nunique()\nFVC_obs = train['FVC'].values\nWeeks = train['Weeks'].values\nPatientID = train['PatientID'].values\n\nX = train[['Weeks', 'Male', 'ExSmoker', 'CurrentlySmokes', \n           'Percent_base_x', '0', '1', '2', '3', '4', '5']].values\n#            , '6', '7', '8', '9']].values\n\nPatientID = train['PatientID'].values\n\nwith pm.Model() as model_c:\n    # create shared variables that can be changed later on\n    FVC_obs_shared = pm.Data(\"FVC_obs_shared\", FVC_obs)\n    X_shared = pm.Data('X_shared', X)\n    PatientID_shared = pm.Data('PatientID_shared', PatientID)\n    \n    mu_a = pm.Normal('mu_a', mu=1700, sigma=400)\n    sigma_a = pm.HalfNormal('sigma_a', 1000.)\n    mu_b = pm.Normal('mu_b', mu=-4., sigma=1., shape=X.shape[1])\n    sigma_b = pm.HalfNormal('sigma_b', 5.)\n\n    a = pm.Normal('a', mu=mu_a, sigma=sigma_a, shape=n_patients)\n    b = pm.Normal('b', mu=mu_b, sigma=sigma_b, shape=(n_patients, X.shape[1]))\n\n    # Model error\n    sigma = pm.HalfNormal('sigma', 150.)\n\n    FVC_est = a[PatientID_shared] + (b[PatientID_shared] * X_shared).sum(axis=1)\n\n    # Data likelihood\n    FVC_like = pm.Normal('FVC_like', mu=FVC_est,\n                         sigma=sigma, observed=FVC_obs_shared)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with model_c:\n    trace_c = pm.sample(2000, tune=2000, target_accept=.90, init=\"adapt_diag\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aux = train.groupby('Patient').first().reset_index()\npred_template = []\nfor i in range(train['Patient'].nunique()):\n    df = pd.DataFrame(columns=['PatientID', 'Weeks'])\n    df['Weeks'] = np.arange(-12, 134)\n    df['PatientID'] = i\n    df['Male'] = aux[aux['PatientID'] == i]['Male'].values[0]\n    df['ExSmoker'] = aux[aux['PatientID'] == i]['ExSmoker'].values[0]\n    df['CurrentlySmokes'] = aux[aux['PatientID'] == i]['CurrentlySmokes'].values[0]\n    df['Percent_base'] = aux[aux['PatientID'] == i]['Percent_base_x'].values[0]\n\n    df['0'] = aux[aux['PatientID'] == i]['0'].values[0]\n    df['1'] = aux[aux['PatientID'] == i]['1'].values[0]\n    df['2'] = aux[aux['PatientID'] == i]['2'].values[0]\n    df['3'] = aux[aux['PatientID'] == i]['3'].values[0]\n    df['4'] = aux[aux['PatientID'] == i]['4'].values[0]\n    df['5'] = aux[aux['PatientID'] == i]['5'].values[0]\n#     df['6'] = aux[aux['PatientID'] == i]['6'].values[0]\n#     df['7'] = aux[aux['PatientID'] == i]['7'].values[0]\n#     df['8'] = aux[aux['PatientID'] == i]['8'].values[0]\n#     df['9'] = aux[aux['PatientID'] == i]['9'].values[0]\n\n    # df['FVC_ref'] = aux[aux['PatientID'] == i]['FVC_ref'].values[0]\n    pred_template.append(df)\npred_template = pd.concat(pred_template, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict posteriors\nwith model_c:\n    pm.set_data({\n        \"PatientID_shared\": pred_template['PatientID'].values.astype(int),\n        \"X_shared\": pred_template[['Weeks', 'Male', 'ExSmoker', \n                                   'CurrentlySmokes', \n                                   'Percent_base','0', '1', '2', '3', '4', '5']].values.astype(int),\n#                                    , '6', '7', '8', '9']].values.astype(int),\n                 \n        \"FVC_obs_shared\": np.zeros(len(pred_template)).astype(int),\n    })\n    post_pred = pm.sample_posterior_predictive(trace_c)\n    \ndf = pd.DataFrame(columns=['Patient', 'Weeks', 'FVC_pred', 'sigma'])\ndf['Patient'] = le_id.inverse_transform(pred_template['PatientID'])\ndf['Weeks'] = pred_template['Weeks']\ndf['FVC_pred'] = post_pred['FVC_like'].T.mean(axis=1)\ndf['sigma'] = post_pred['FVC_like'].T.std(axis=1)\ndf['FVC_inf'] = df['FVC_pred'] - df['sigma']\ndf['FVC_sup'] = df['FVC_pred'] + df['sigma']\ndf = pd.merge(df, train[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])\ndf = df.rename(columns={'FVC': 'FVC_true'})\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(columns=['Patient', 'Weeks', 'Patient_Week', 'FVC', 'Confidence'])\ndf['Patient'] = pred_template['PatientID']\ndf['Weeks'] = pred_template['Weeks']\ndf['Patient_Week'] = df['Patient'].astype(str) + '_' + df['Weeks'].astype(str)\ndf['FVC'] = post_pred['FVC_like'].T.mean(axis=1)\ndf['Confidence'] = post_pred['FVC_like'].T.std(axis=1)\nfinal = df[['Patient_Week', 'FVC', 'Confidence']]\n\nfinal.to_csv('submission.csv', index=False)\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}