{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Osic CNN Pytorch"},{"metadata":{},"cell_type":"markdown","source":"# Goal\n\nThe goal of this notebook is to try PyTorch Convolutional Neural Network for OSIC pulmonary fibrosis.<br>\n<br>\nFor a starter notebook click [here](https://www.kaggle.com/twinkle0705/your-starter-notebook-for-osic)\n<br>\nData preparation was used from this [notebook](https://www.kaggle.com/ulrich07/osic-multiple-quantile-regression-starter)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.utils.data import TensorDataset, DataLoader\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_absolute_error\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"PATH = \"../input/osic-pulmonary-fibrosis-progression\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(f\"{PATH}/train.csv\")\n\n# drop duplicates for patient and weeks\ndf_train.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(f\"{PATH}/test.csv\")\n\n# prepare submission\ndf_sub = pd.read_csv(f\"{PATH}/sample_submission.csv\")\ndf_sub['Patient'] = df_sub['Patient_Week'].apply(lambda x:x.split('_')[0])\ndf_sub['Weeks'] = df_sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\ndf_sub = df_sub[['Patient', 'Weeks', 'Confidence', 'Patient_Week']]\ndf_sub = df_sub.merge(df_test.drop('Weeks', axis=1), on='Patient')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['FROM'] = 'train'\ndf_test['FROM'] = 'val'\ndf_sub['FROM'] = 'test'\ndata = df_train.append([df_test, df_sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.FROM == 'test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize\n\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = data.loc[data.FROM == 'train']\ndf_test = data.loc[data.FROM == 'val']\ndf_sub = data.loc[data.FROM == 'test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df_train['FVC'].values\nz = df_train[FE].values\nze = df_sub[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_y = y.max()\ny = y / max_y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pytorch Module"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = nn.Sequential(nn.Conv1d(3, 32, 3),\n                      nn.Dropout(p=0.42),\n                      nn.ReLU(),\n                      nn.Conv1d(32, 64, 1),\n                      nn.Dropout(p=0.35),\n                      nn.Flatten(),\n                      nn.Linear(64, 128),\n                      nn.Tanh(),\n                      nn.Linear(128, 64),\n                      nn.Tanh(),\n                      nn.Linear(64, 32),\n                      nn.Tanh(),\n                      nn.Linear(32, 16),\n                      nn.Tanh(),\n                      nn.Linear(16, 8),\n                      nn.Tanh(),\n                      nn.Linear(8, 1))\n\nmodel = model.to('cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = KFold(n_splits=5)\nkf2 = KFold(n_splits=128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_fn = torch.nn.MSELoss(reduction='sum')\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0005)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"losses = []\nfor tr_idx, val_idx in tqdm(kf.split(z)):\n    \n    for _ in tqdm(range(300), position=0, leave=True):\n        for tidx, vidx in kf2.split(tr_idx):\n            optimizer.zero_grad()\n\n            y_pred = model(torch.Tensor(z[tidx]).reshape(len(z[tidx]), 3, 3).to('cuda'))\n            loss = loss_fn(y_pred, torch.Tensor(y[tidx]).reshape(len(y[tidx]), 1).to('cuda'))\n            losses.append(loss)\n\n            loss.backward()\n            optimizer.step()\n        \n    pred[val_idx] = model(torch.Tensor(z[val_idx]).reshape(len(z[val_idx]), 3, 3).to('cuda')).cpu().detach().numpy()\n    pe += (model(torch.Tensor(ze).reshape(len(ze), 3, 3).to('cuda')) / 5).cpu().detach().numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot Losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(losses)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred, pe = pred * max_y, pe * max_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub['FVC1'] = pe[:, 1]\ndf_sub['Confidence1'] = pe[:, 2] - pe[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = df_sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Conclusion\n\nSince the high position on the submission, either CNN is not the way to go or the layers need to be tweaked."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}