{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OSIC Pulmonary Fibrosis Progression","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm import tqdm\nimport pydicom\n\nfrom os import listdir\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nfrom torchvision import transforms\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.models as tmodels\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\n\ntorch.manual_seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:37.474679Z","iopub.execute_input":"2022-01-15T09:41:37.475205Z","iopub.status.idle":"2022-01-15T09:41:40.238473Z","shell.execute_reply.started":"2022-01-15T09:41:37.475060Z","shell.execute_reply":"2022-01-15T09:41:40.237713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GPU\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(\"cuda\" if use_cuda else \"cpu\")\nif use_cuda == False:\n    print(\"WARNING: CPU will be used for training.\")\nelse:\n    print(\"GPU enabled\")\n    \nBATCH_SIZE = 1\n#device = torch.device(\"cpu\"); print(\"WARNING: CPU will be used for training.\")#","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:40.240352Z","iopub.execute_input":"2022-01-15T09:41:40.240601Z","iopub.status.idle":"2022-01-15T09:41:40.292237Z","shell.execute_reply.started":"2022-01-15T09:41:40.240569Z","shell.execute_reply":"2022-01-15T09:41:40.291455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntest_data = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\n\nbad_ids = ['ID00011637202177653955184', 'ID00052637202186188008618',# gdcm\n          'ID00026637202179561894768', 'ID00078637202199415319443', 'ID00128637202219474716089', 'ID00132637202222178761324', # bad window\n          'ID00214637202257820847190', 'ID00242637202264759739921', 'ID00248637202266698862378']                              # bad window\nfor bad_id in bad_ids: \n    train_data.drop(train_data[train_data.Patient == bad_id].index, inplace=True)\n    \n# test ids are in train table (wtf??)\ntest_ids = test_data.Patient.unique()\ncheat_results = train_data[train_data.Patient.isin(test_ids)]\n\nfor bad_id in test_ids: \n    train_data.drop(train_data[train_data.Patient == bad_id].index, inplace=True)\n    \n# train data \ntrain_data.drop_duplicates(subset=['Patient', 'Weeks'], keep=False, inplace=True)\n\ntrain_data['Sex'] = train_data['Sex'].map({'Male': 0,'Female': 1})\ntrain_data['SmokingStatus'] = train_data['SmokingStatus'].map({'Never smoked': -1,\n                                                     'Ex-smoker': 0, \n                                                     'Currently smokes': 1})\navg_age = 67\nmax_age = 100\ntrain_data['Age'] = train_data['Age'] / max_age\n\n\n# test data \ntest_data['Sex'] = test_data['Sex'].map({'Male': 0,'Female': 1})\ntest_data['SmokingStatus'] = test_data['SmokingStatus'].map({'Never smoked': -1,\n                                                     'Ex-smoker': 0, \n                                                     'Currently smokes': 1})\navg_age = 67\nmax_age = 100\ntest_data['Age'] = test_data['Age'] / max_age\n\ntest_data['ImagePath'] = '../input/preprocessed-scans-osic-pfp/' + test_data.Patient + '.npy'","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:40.293798Z","iopub.execute_input":"2022-01-15T09:41:40.294315Z","iopub.status.idle":"2022-01-15T09:41:40.366549Z","shell.execute_reply.started":"2022-01-15T09:41:40.294276Z","shell.execute_reply":"2022-01-15T09:41:40.365750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Percent is ignored\n\npatient_id = []\nage = []\nsex = []\nsmoking_status = []\n\nslope = []\nintercept = []\n\nfirst_week = []\nfirst_fvc = []\n\nimage_path = []\n\npatients = train_data.Patient.unique()[:]\nfor patient in patients:\n    \n    patient_id.append(patient)\n    patient_table = train_data[train_data.Patient == patient].sort_values(by=['Weeks'])\n        \n    fvc = patient_table.FVC.to_numpy()\n    weeks = patient_table.Weeks.to_numpy()\n    \n    age.append(patient_table.Age.values[0])\n    sex.append(patient_table.Sex.values[0])\n    smoking_status.append(patient_table.SmokingStatus.values[0])\n    \n    first_week.append(patient_table.Weeks.values[0])\n    first_fvc.append(patient_table.FVC.values[0])\n    \n    eq = np.vstack([weeks, np.ones(len(weeks))]).T\n    slope_, intercept_ = np.linalg.lstsq(eq, fvc, rcond=None)[0]\n    \n    slope.append(slope_)\n    intercept.append(intercept_)\n    \n    image_path.append('../input/preprocessed-scans-osic-pfp/' + patient + '.npy')\n    \n\npatients_data = pd.DataFrame(data=patient_id, columns=[\"Patient\"])\npatients_data.loc[:, \"Age\"] = age\npatients_data.loc[:, \"Sex\"] = sex\npatients_data.loc[:, \"SmokingStatus\"] = smoking_status\npatients_data.loc[:, \"Weeks\"] = first_week\npatients_data.loc[:, \"FVC\"] = first_fvc\npatients_data.loc[:, \"Slope\"] = slope\npatients_data.loc[:, \"Intercept\"] = intercept\npatients_data.loc[:, \"ImagePath\"] = image_path\n\n# Normalize Slope in [0, 1] min:-30, max:15 \npatients_data.Slope = (patients_data.Slope + 30) / 45 \n\nprint('patients_data shape:', patients_data.shape)\npatients_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:40.368675Z","iopub.execute_input":"2022-01-15T09:41:40.368945Z","iopub.status.idle":"2022-01-15T09:41:40.637912Z","shell.execute_reply.started":"2022-01-15T09:41:40.368907Z","shell.execute_reply":"2022-01-15T09:41:40.637080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PulmonaryFibrosisProgressionDataset(Dataset):\n    \n    def __init__(\n            self,\n            df,\n            train: bool = False,\n            test: bool = False,\n            transform = None,\n    ) -> None:\n         \n        self.train = train\n        self.test = test\n\n        self.df = df\n        \n        self.data_features = ['Age', 'Sex', 'SmokingStatus', 'ImagePath']\n        self.target_features = ['Slope']#, 'Intercept']\n        \n        \n        self.transform = transform\n        \n    \n    def __getitem__(self, index: int):\n        \n        patient_data = self.df.iloc[index]\n        \n        dcm_path = '../input/osic-pulmonary-fibrosis-progression/train/{}/'.format('ID00007637202177411956430')\n\n        files = listdir(dcm_path)\n        file_nums = [np.int(each_file.split(\".\")[0]) for each_file in files]\n        sorted_file_nums = np.sort(file_nums)[::-1]\n\n        tensor_images = torch.zeros((3, 512, 512))\n        #tensor_images[0] = self.transform(pydicom.dcmread(dcm_path + str(sorted_file_nums[len(sorted_file_nums)//2 - 1]) + \".dcm\" ).pixel_array.astype(np.float32))\n        #tensor_images[1] = self.transform(pydicom.dcmread(dcm_path + str(sorted_file_nums[len(sorted_file_nums)//2    ]) + \".dcm\" ).pixel_array.astype(np.float32))\n        #tensor_images[1] = self.transform(pydicom.dcmread(dcm_path + str(sorted_file_nums[len(sorted_file_nums)//2 + 1]) + \".dcm\" ).pixel_array.astype(np.float32))\n        \n        \n        #loaded_images = np.load(patient_data['ImagePath'])\n        #tensor_images = torch.zeros((1, 512, 512))\n        #\n        #for i in [0,1,2]:\n        #    tensor_images[i] = self.transform(-1*loaded_images[4+i])\n            \n        \n        tabular_data = patient_data[self.data_features[:-1]].to_numpy(dtype=float)\n        \n        if self.train:\n            target = patient_data[self.target_features].to_numpy(dtype=float)\n            \n        elif self.test:\n            target = np.zeros(2, dtype=float)\n\n        return tensor_images, tabular_data, target\n    \n    def __len__(self) -> int:\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:49:00.338939Z","iopub.execute_input":"2022-01-15T09:49:00.339218Z","iopub.status.idle":"2022-01-15T09:49:00.349604Z","shell.execute_reply.started":"2022-01-15T09:49:00.339187Z","shell.execute_reply":"2022-01-15T09:49:00.348695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n                transforms.ToTensor(),\n                #transforms.Normalize(-2000, 5000),\n                #transforms.RandomAffine(degrees=(-5, 5), translate=(0.0, 0.0))\n             ])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:49:08.616907Z","iopub.execute_input":"2022-01-15T09:49:08.617189Z","iopub.status.idle":"2022-01-15T09:49:08.620833Z","shell.execute_reply.started":"2022-01-15T09:49:08.617147Z","shell.execute_reply":"2022-01-15T09:49:08.620146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = PulmonaryFibrosisProgressionDataset(patients_data, train=True, transform=transform)\n\na.__getitem__(0)[0].shape","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:38:56.404311Z","iopub.execute_input":"2022-01-15T09:38:56.405037Z","iopub.status.idle":"2022-01-15T09:38:56.427411Z","shell.execute_reply.started":"2022-01-15T09:38:56.405006Z","shell.execute_reply":"2022-01-15T09:38:56.426475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = a.__getitem__(0)[0].permute(1, 2, 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:38:56.646291Z","iopub.execute_input":"2022-01-15T09:38:56.646748Z","iopub.status.idle":"2022-01-15T09:38:56.661969Z","shell.execute_reply.started":"2022-01-15T09:38:56.646717Z","shell.execute_reply":"2022-01-15T09:38:56.660739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(image.flatten())","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:38:57.543162Z","iopub.execute_input":"2022-01-15T09:38:57.543494Z","iopub.status.idle":"2022-01-15T09:38:58.844321Z","shell.execute_reply.started":"2022-01-15T09:38:57.543453Z","shell.execute_reply":"2022-01-15T09:38:58.843411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class ModelM3(nn.Module):\n    def __init__(self):\n        super(ModelM3, self).__init__()\n    \n        final_image_width = 10\n        features_length = 3\n        concat_length = final_image_width + features_length\n        \n        self.densenet = tmodels.densenet161(pretrained=True)\n        self.fc1 = nn.Linear(1000, final_image_width)\n        self.fc2 = nn.Linear(concat_length, concat_length)\n        self.fc3 = nn.Linear(concat_length, 1)\n        \n        \n    def forward(self, x):\n        \n        image = x[0]\n        data = x[1]\n        #target_length = len(x[2][0])\n                \n        result = self.densenet(image)\n        result = torch.tanh(self.fc1(result))\n        result = torch.cat((result, data), dim=1)\n        result = nn.Dropout()(result)\n        result = torch.tanh(self.fc2(result))\n        result = torch.tanh(self.fc3(result))\n        \n        \n        return result","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:47.449017Z","iopub.execute_input":"2022-01-15T09:41:47.449471Z","iopub.status.idle":"2022-01-15T09:41:47.457276Z","shell.execute_reply.started":"2022-01-15T09:41:47.449434Z","shell.execute_reply":"2022-01-15T09:41:47.456548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_n_params(model):\n    np=0\n    for p in list(model.parameters()):\n        np += p.nelement()\n    return np","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:47.892705Z","iopub.execute_input":"2022-01-15T09:41:47.893226Z","iopub.status.idle":"2022-01-15T09:41:47.897262Z","shell.execute_reply.started":"2022-01-15T09:41:47.893189Z","shell.execute_reply":"2022-01-15T09:41:47.896561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EMA:\n    def __init__(self, model, decay):\n        self.decay = decay\n        self.shadow = {}\n        self.original = {}\n\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                self.shadow[name] = param.data.clone()\n\n    def __call__(self, model, num_updates):\n        decay = min(self.decay, (1.0 + num_updates) / (10.0 + num_updates))\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                assert name in self.shadow\n                new_average = (1.0 - decay) * param.data + decay * self.shadow[name]\n                self.shadow[name] = new_average.clone()\n\n    def assign(self, model):\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                assert name in self.shadow\n                self.original[name] = param.data.clone()\n                param.data = self.shadow[name]\n\n    def resume(self, model):\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                assert name in self.shadow\n                param.data = self.original[name]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:48.258270Z","iopub.execute_input":"2022-01-15T09:41:48.258726Z","iopub.status.idle":"2022-01-15T09:41:48.276575Z","shell.execute_reply.started":"2022-01-15T09:41:48.258682Z","shell.execute_reply":"2022-01-15T09:41:48.275054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = F.l1_loss\nloss_points = {}\nloss_points['train'] = []\nloss_points['test'] = []\n\ndef train(epoch, model):\n    model.train()\n    \n    g_step = 0\n\n    for batch_idx, (image, data, target) in enumerate(train_loader):\n        # send to device\n        image, data, target = image.to(device), data.float().to(device), target.float().to(device)\n        \n        optimizer.zero_grad()\n        output = model([image, data, target])\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        \n        loss_points['train'].append(loss.item())\n        \n        g_step += 1\n        ema(model, g_step)\n        \n        if batch_idx % 20 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}\\t output: {:.3f},\\t target: {:.3f}'.format(\n                epoch+1, (batch_idx+1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx+1) / len(train_loader), loss.item()**2, output[0][0], target[0][0]))\n                \ndef test(model):\n    model.eval()\n    ema.assign(model)\n    test_loss = 0\n    predictions = []\n    targets = []\n    \n    with torch.no_grad():\n        for image, data, target in test_loader:\n            # send to device\n            image, data, target = image.to(device), data.float().to(device), target.float().to(device)\n\n            output = model([image, data, target])\n            targets.append(target)\n            \n            loss_points['test'].append(criterion(output, target).item())\n            \n            test_loss += criterion(output, target, reduction='sum').item()**2 # sum up batch loss                                                               \n            predictions.append(output)\n            \n            print('output: {:.3f},\\t target: {:.3f}'.format(output[0][0], target[0][0]))\n    \n    ema.resume(model)\n    \n    outputs = np.array([predictions[i].item() for i in range(len(predictions))], dtype=float)\n    targets = np.array([targets[i].item() for i in range(len(targets))], dtype=float)\n        \n    test_loss /= len(test_loader.dataset)\n    correlation = np.corrcoef(outputs, targets)[0][1]\n    print('\\nTest set: Average loss: {:.4f}\\t Correlation: {:.4f}\\n'.format(test_loss, correlation))\n    \n    return test_loss\n\n\ndef predict(model):\n    \n    model.eval()\n    #ema.assign(model)\n    predictions = {}\n    \n    with torch.no_grad():\n        for i, (image, data, target) in enumerate(prediction_loader):\n            \n            print('Predicting patient:', patients_testing_data.iloc[i].Patient)\n            # send to device\n            image, data, target = image.to(device), data.float().to(device), target.float().to(device)\n\n            output = model([image, data, target])\n            \n            predictions[patients_testing_data.iloc[i].Patient] = output.item()\n            \n    #ema.resume(model)\n    \n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:48.856182Z","iopub.execute_input":"2022-01-15T09:41:48.856618Z","iopub.status.idle":"2022-01-15T09:41:48.876042Z","shell.execute_reply.started":"2022-01-15T09:41:48.856581Z","shell.execute_reply":"2022-01-15T09:41:48.875398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma, 70) # changed from 70, trie 66.7 too\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta, 1000)\n    sq2 = np.sqrt(2)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)\n    return np.mean(metric)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:49.565685Z","iopub.execute_input":"2022-01-15T09:41:49.566389Z","iopub.status.idle":"2022-01-15T09:41:49.572182Z","shell.execute_reply.started":"2022-01-15T09:41:49.566350Z","shell.execute_reply":"2022-01-15T09:41:49.570670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold","metadata":{}},{"cell_type":"code","source":"#transform = transforms.Compose([\n#                transforms.ToTensor(),\n#                transforms.Normalize(572.5, 255.),\n#                #transforms.RandomAffine(degrees=(-5, 5), translate=(0.0, 0.0))\n#             ])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:51.078051Z","iopub.execute_input":"2022-01-15T09:41:51.078732Z","iopub.status.idle":"2022-01-15T09:41:51.082494Z","shell.execute_reply.started":"2022-01-15T09:41:51.078695Z","shell.execute_reply":"2022-01-15T09:41:51.081346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = KFold(n_splits=5, shuffle=True)\nkfold_split = kfold.split(patients_data[:])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:49:17.944591Z","iopub.execute_input":"2022-01-15T09:49:17.945117Z","iopub.status.idle":"2022-01-15T09:49:17.948870Z","shell.execute_reply.started":"2022-01-15T09:49:17.945077Z","shell.execute_reply":"2022-01-15T09:49:17.948203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 30\nema_decay = 0.999\n\nmodel = ModelM3()\ninitial_state = model.state_dict()\n\nbest_state = initial_state\nbest_fold = 0\ntest_loss = 100\nbest_test_ids = 0","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:41:52.562329Z","iopub.execute_input":"2022-01-15T09:41:52.562862Z","iopub.status.idle":"2022-01-15T09:41:58.340050Z","shell.execute_reply.started":"2022-01-15T09:41:52.562823Z","shell.execute_reply":"2022-01-15T09:41:58.339327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(initial_state)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:49:20.633410Z","iopub.execute_input":"2022-01-15T09:49:20.633673Z","iopub.status.idle":"2022-01-15T09:49:20.785534Z","shell.execute_reply.started":"2022-01-15T09:49:20.633642Z","shell.execute_reply":"2022-01-15T09:49:20.784885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_ids, test_ids) in enumerate(kfold_split):\n\n    patients_training_data = patients_data.iloc[train_ids]\n    patients_testing_data  = patients_data.iloc[test_ids]\n\n    if fold == 0:\n        print('Train samples:', len(patients_training_data))\n        print('Test samples:', len(patients_testing_data))\n\n    \n    print('-'*100)\n    print('Fold number:', fold)\n    \n    \n    train_loader = torch.utils.data.DataLoader(\n        PulmonaryFibrosisProgressionDataset(patients_training_data, train=True, transform=transform),\n        batch_size=BATCH_SIZE, shuffle=True)\n\n    test_loader = torch.utils.data.DataLoader(\n        PulmonaryFibrosisProgressionDataset(patients_testing_data, train=True, transform=transform),\n        batch_size=BATCH_SIZE)\n\n    model = ModelM3()\n    model.load_state_dict(initial_state)\n    model.to(device)\n\n    optimizer = optim.Adam(model.parameters(), lr=0.01)\n\n    ema = EMA(model, decay=ema_decay)\n\n\n    for epoch in range(n_epochs):\n\n        train(epoch, model)\n        current_loss = test(model)\n        \n        if current_loss < test_loss:\n            test_loss = current_loss\n            best_state = model.state_dict()\n            best_fold = fold\n            best_test_ids = test_ids\n\n    print('\\n\\n')\n\n    \n    break\n\nprint('Best fold:', best_fold)\ntorch.save(best_state, '../working/best_state.pt')\nprint('Done!')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:49:22.141237Z","iopub.execute_input":"2022-01-15T09:49:22.141905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictions","metadata":{}},{"cell_type":"code","source":"patients_testing_data  = test_data#.iloc[best_test_ids]\n\nprediction_loader = torch.utils.data.DataLoader(\n    PulmonaryFibrosisProgressionDataset(patients_testing_data, test=True, transform=transform),\n    batch_size=BATCH_SIZE)\n\nmodel = ModelM3()\nmodel.load_state_dict(torch.load('../input/best-state/best_state.pt'))\nmodel.to(device)\n\nema = EMA(model, decay=ema_decay)\n\n#test(model)\npredictions = predict(model)\n\n\n\n\nprint('\\n\\n')\nprint('Done!')","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:46:55.064264Z","iopub.execute_input":"2022-01-12T08:46:55.065186Z","iopub.status.idle":"2022-01-12T08:47:03.145492Z","shell.execute_reply.started":"2022-01-12T08:46:55.06514Z","shell.execute_reply":"2022-01-12T08:47:03.144759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:47:04.888963Z","iopub.execute_input":"2022-01-12T08:47:04.889357Z","iopub.status.idle":"2022-01-12T08:47:04.898106Z","shell.execute_reply.started":"2022-01-12T08:47:04.889305Z","shell.execute_reply":"2022-01-12T08:47:04.897246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = np.array(list(predictions.values()))","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:47:07.813767Z","iopub.execute_input":"2022-01-12T08:47:07.814038Z","iopub.status.idle":"2022-01-12T08:47:07.81801Z","shell.execute_reply.started":"2022-01-12T08:47:07.81401Z","shell.execute_reply":"2022-01-12T08:47:07.817275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(a.mean()-a.std(), a.mean()+a.std())","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:47:08.107711Z","iopub.execute_input":"2022-01-12T08:47:08.108097Z","iopub.status.idle":"2022-01-12T08:47:08.11345Z","shell.execute_reply.started":"2022-01-12T08:47:08.108064Z","shell.execute_reply":"2022-01-12T08:47:08.11251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(patients_data.Slope.values, bins=200)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:47:08.391735Z","iopub.execute_input":"2022-01-12T08:47:08.391984Z","iopub.status.idle":"2022-01-12T08:47:09.001044Z","shell.execute_reply.started":"2022-01-12T08:47:08.391958Z","shell.execute_reply":"2022-01-12T08:47:09.000303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(a, bins=200)\nplt.xlim(0, 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:47:10.234131Z","iopub.execute_input":"2022-01-12T08:47:10.234654Z","iopub.status.idle":"2022-01-12T08:47:11.332742Z","shell.execute_reply.started":"2022-01-12T08:47:10.234611Z","shell.execute_reply":"2022-01-12T08:47:11.332005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post processing","metadata":{}},{"cell_type":"code","source":"results = pd.DataFrame(columns=['Patient_week', 'FVC', 'Confidence'])\nfor patient, slope in predictions.items():\n    \n    true_fvc = test_data[test_data.Patient == patient].FVC.values[0]\n    true_weeks = test_data[test_data.Patient == patient].Weeks.values[0]\n\n    slope = slope*45 - 30\n    intercept = true_fvc - slope*true_weeks\n    \n    predicted_weeks = list(range(-12, 134))\n    predicted_fvc = [slope*weeks + intercept for weeks in predicted_weeks]\n    \n    cheat_results_weeks = cheat_results[cheat_results.Patient == patient].Weeks.values\n    cheat_results_fvc = cheat_results[cheat_results.Patient == patient].FVC.values\n    \n    \n    patient_week = [patient+'_'+str(week) for week in predicted_weeks]\n    \n    for i in range(len(patient_week)):\n        results = results.append({'Patient_week': patient_week[i], \n                                  'FVC': predicted_fvc[i], \n                                  'Confidence': 70\n                                 }, ignore_index=True)\n    \n    \n    #to_be_appended = np.array([patient_week, predicted_fvc]).T\n    #print(to_be_appended)\n    \n    \n    #sns.scatterplot(x=predicted_weeks, y=predicted_fvc)\n    #sns.scatterplot(x=cheat_results_weeks, y=cheat_results_fvc)\n\n    \nresults","metadata":{"execution":{"iopub.status.busy":"2022-01-12T09:10:24.052425Z","iopub.execute_input":"2022-01-12T09:10:24.052868Z","iopub.status.idle":"2022-01-12T09:10:25.482735Z","shell.execute_reply.started":"2022-01-12T09:10:24.052828Z","shell.execute_reply":"2022-01-12T09:10:25.482066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv('../working/submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T09:13:26.520986Z","iopub.execute_input":"2022-01-12T09:13:26.521607Z","iopub.status.idle":"2022-01-12T09:13:26.533099Z","shell.execute_reply.started":"2022-01-12T09:13:26.521566Z","shell.execute_reply":"2022-01-12T09:13:26.532244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(x=range(len(loss_points['train'])), y=loss_points['train'])","metadata":{"execution":{"iopub.status.busy":"2022-01-14T13:10:51.111708Z","iopub.execute_input":"2022-01-14T13:10:51.112049Z","iopub.status.idle":"2022-01-14T13:10:51.33708Z","shell.execute_reply.started":"2022-01-14T13:10:51.112009Z","shell.execute_reply":"2022-01-14T13:10:51.336342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(x=range(len(loss_points['test'])), y=loss_points['test'])","metadata":{"execution":{"iopub.status.busy":"2022-01-14T13:10:51.33845Z","iopub.execute_input":"2022-01-14T13:10:51.338708Z","iopub.status.idle":"2022-01-14T13:10:51.539012Z","shell.execute_reply.started":"2022-01-14T13:10:51.338674Z","shell.execute_reply":"2022-01-14T13:10:51.538345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir(tmodels)","metadata":{"execution":{"iopub.status.busy":"2022-01-14T13:37:06.235224Z","iopub.execute_input":"2022-01-14T13:37:06.236081Z","iopub.status.idle":"2022-01-14T13:37:06.245569Z","shell.execute_reply.started":"2022-01-14T13:37:06.236047Z","shell.execute_reply":"2022-01-14T13:37:06.243907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:53:29.985473Z","iopub.execute_input":"2022-01-15T08:53:29.985773Z","iopub.status.idle":"2022-01-15T08:53:29.992519Z","shell.execute_reply.started":"2022-01-15T08:53:29.985742Z","shell.execute_reply":"2022-01-15T08:53:29.99115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}