{"cells":[{"metadata":{},"cell_type":"markdown","source":"# I have found a way to combine a way to combine 3d images with the tabular data for thefinal prediction"},{"metadata":{},"cell_type":"markdown","source":"While forking run first to cell will restart the kernel and continue running from the 3rd cell."},{"metadata":{"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display_html\ndef restartkernel() :\n    display_html(\"<script>Jupyter.notebook.kernel.restart()</script>\",raw=True)\nrestartkernel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gdcm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Importing Libraries **"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import plotly.graph_objs as go\nimport pydicom as dicom\nimport matplotlib.pyplot as plt\nimport torch.nn as nn\nimport cv2\nimport torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math\nfrom sklearn import preprocessing\nfrom sklearn.compose import ColumnTransformer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading files"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"folder_path = '../input/osic-pulmonary-fibrosis-progression'\ntrain_csv = folder_path + '/train.csv'\ntest_csv = folder_path+ '/test.csv'\nsample_csv = folder_path + '/sample_submission.csv'\n\ntrain_data = pd.read_csv(train_csv)\ntest_data = pd.read_csv(test_csv)\nsample = pd.read_csv(sample_csv)\n\nprint(train_data.shape)\nprint(test_data.shape)\nprint(sample.shape)\n\ntrain_data.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cleaning Tabular data"},{"metadata":{},"cell_type":"markdown","source":"## Below patient have lot of variance in their FVC so I have decided to drop those Patients data"},{"metadata":{},"cell_type":"markdown","source":"I have observed lot of varince in the FVC of few patients so I have decided to drop them"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped=train_data.groupby('Patient')\npid=[]\nfor name,group in grouped:\n    if np.var(group.FVC)>100000:\n        pid.append(name)\n        print(name)\n        print(np.var(group.FVC),np.std(group.FVC))\n        \nfig =go.Figure()\n\nfor patient in pid:\n    df = train_data[train_data[\"Patient\"] == patient]\n    fig.add_trace(go.Scatter(x=df.Weeks,y=df.FVC,\n                            mode='lines',\n                            name=str(patient)))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dropping the above Patients"},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train_data\nfor patient in pid:\n    train=train[train[\"Patient\"] != patient]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Displaying FVC of patients after Dropping "},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped=train_data.groupby('Patient')\npid=[]\nfor name,group in grouped:\n    pid.append(name)\n        \nfig =go.Figure()\n\nfor patient in pid:\n    df = train_data[train_data[\"Patient\"] == patient]\n    fig.add_trace(go.Scatter(x=df.Weeks,y=df.FVC,\n                            mode='lines',\n                            name=str(patient)))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For Preprocessing of data\nCredits: https://www.kaggle.com/jaideepvalani/updated-pytorch-osic-starter-6-88-6-91"},{"metadata":{},"cell_type":"markdown","source":"cleaning the submission data"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk=test_data\n\nprint(\"add infos\")\nsub =sample\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nprint(sub.index.size)\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(test_data.drop('Weeks', axis=1), on=\"Patient\")\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Combining all the data to perform same type of transformation on train,test, submission dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr=train\ntr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Getting the min weeks(first record) of every patient"},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\ndata.loc[data.Weeks == data.min_week]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Getting the min FVC(first FVC recorded) of every patient."},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\nbase[base.Patient=='ID00419637202311204720264']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\n#del base\ndata['diff_fvc_prev']=data['FVC'].diff(1)/data['FVC'].shift(1)\ndata[data.Patient=='ID00007637202177411956430']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalising Tabular data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\n\nFE += ['age','percent','week','BASE']\ndata[FE]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Filling Null values"},{"metadata":{"trusted":true},"cell_type":"code","source":"data=data.fillna(0)\n\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Splitting the combined data back to train,test,submission data"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Processing Image files"},{"metadata":{"trusted":true},"cell_type":"code","source":"count=0\nfor dirname, _, filenames in os.walk('../input/osic-pulmonary-fibrosis-progression/test'):\n    for filename in filenames:\n        count=count+1\nprint(\"Count of total files in test data:\",count)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finding the shape of each image of a patient and the no of files related to a patient"},{"metadata":{"trusted":true},"cell_type":"code","source":"patients = train['Patient'].unique()\ndata_dir = '../input/osic-pulmonary-fibrosis-progression' + '/train/'\n\nfor patient in patients[0:5]:\n    #patient='ID00026637202179561894768'\n    #label = labels_df.get_value(patient, 'cancer')\n    path = data_dir + patient\n\n    # a couple great 1-liners from: https://www.kaggle.com/gzuidhof/data-science-bowl-2017/full-preprocessing-tutorial\n    slices = [dicom.dcmread(path + '/' + s) for s in os.listdir(path)]\n    try:\n        slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    except:\n        print(patient,slices.ImagePositionPatient[2])\n        break\n    print(patient,slices[0].pixel_array.shape, len(slices))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samp=['ID00009637202177434476278','ID00014637202177757139317']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Viewing the ct scans of one of the patient"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_PX_SIZE = 150\n\nfor patient in patients[:1]:\n    \n    path = data_dir + patient\n    slices = [dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    fig = plt.figure()\n    for num,each_slice in enumerate(slices[:12]):\n        y = fig.add_subplot(3,4,num+1)\n        new_img = cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE))\n        y.imshow(new_img,cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing of the image dataset"},{"metadata":{},"cell_type":"markdown","source":"## Step involved:\n\n- Converting pixel data to HU( a measurement used in CT scans)\n- Chunking the overall images in to 10 images for each patient:\n    we can see that every patient has several different counts of CT scans (patient1-30, patient2-394,..) I want these count to be same inorder to feed to the neural network.\n- Normalising the above values\n- Zero centering the above pixels\n- Resizin the image: all the images are of several different shapes so I resized them to a 64X64 images."},{"metadata":{"trusted":true},"cell_type":"code","source":"MIN_BOUND = -1000.0\nMAX_BOUND = 400.0\n    \n\ndef get_pixels_hu(slices):\n    image = np.stack([s.pixel_array for s in slices])\n    # Convert to int16 (from sometimes int16), \n    # should be possible as values should always be low enough (<32k)\n    image = image.astype(np.int16)\n\n    # Set outside-of-scan pixels to 0\n    # The intercept is usually -1024, so air is approximately 0\n    image[image == -2000] = 0\n    \n    # Convert to Hounsfield units (HU)\n    for slice_number in range(len(slices)):\n        \n        intercept = slices[slice_number].RescaleIntercept\n        slope = slices[slice_number].RescaleSlope\n        \n        if slope != 1:\n            image[slice_number] = slope * image[slice_number].astype(np.float64)\n            image[slice_number] = image[slice_number].astype(np.int16)\n            \n        image[slice_number] += np.int16(intercept)\n    return np.array(image, dtype=np.int16)\n\n\n\ndef normalize(image):\n    image = (image - MIN_BOUND) / (MAX_BOUND - MIN_BOUND)\n    image[image>1] = 1.\n    image[image<0] = 0.\n    return image\n\n\nPIXEL_MEAN = 0.25\ndef zero_center(image):\n    image = image - PIXEL_MEAN\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\ndef process_data(patient,data_dir,img_px_size=64, HM_SLICES=10, visualize=False,):\n    def chunks(l, n):\n    # Credit: Ned Batchelder\n    # Link: http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks\n   \n        for i in range(0, len(l), n):\n            yield l[i:i + n]\n\n    def mean(l):\n        return sum(l) / len(l)\n    #patient='ID00422637202311677017371'\n    \n    path = data_dir + patient\n    slices = [dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    try:\n        print(patient)\n        # sorting the ct scan of a patient based on the 3D position of the scan\n        slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    except:\n        print(patient,\"No Image position patient\")\n        return []\n        \n    new_slices = []\n    # 1-Converting pixel data to HU\n    slices= get_pixels_hu(slices)\n    # 2-Chunking the overall images in to 10 images for each patient\n    chunk_sizes = round(len(slices) / HM_SLICES)\n    if(len(slices)%HM_SLICES!=0):\n        a=math.floor((len(slices) / HM_SLICES))\n        b=math.ceil(len(slices) / HM_SLICES)\n        x=abs((len(slices)-(b*HM_SLICES))/(b-a))\n        y=HM_SLICES - x         \n\n        split=int(x*a)\n\n        for slice_chunk in chunks(slices[:split], a):\n            slice_chunk = list(map(mean, zip(*slice_chunk)))\n            new_slices.append(slice_chunk)\n        for slice_chunk in chunks(slices[split:], b):\n            slice_chunk = list(map(mean, zip(*slice_chunk)))\n            new_slices.append(slice_chunk)\n    else:\n        chunk_sizes = round(len(slices) / HM_SLICES)\n        for slice_chunk in chunks(slices, chunk_sizes):\n            slice_chunk = list(map(mean, zip(*slice_chunk)))\n            #3-Normalising the values of images(slices)\n            b=normalize(np.array(slice_chunk))\n            #4-Zero centering the above pixels\n            slice_chunk=zero_center(b)\n            new_slices.append(slice_chunk)\n    real_slices = []\n    fig = plt.figure()\n    for num,each_slice in enumerate(new_slices):\n        #5- Resizing the Images\n        each_slice = cv2.resize(np.array(each_slice),(IMG_PX_SIZE,IMG_PX_SIZE))\n        real_slices.append(each_slice)\n        #print(np.array(each_slice).shape)\n        y = fig.add_subplot(4,5,num+1)\n        y.imshow(each_slice, cmap='gray')\n    plt.show()\n    return np.array(real_slices),\n\nIMG_PX_SIZE = 64\nHM_SLICES = 64\n\ndata_dir = '../input/osic-pulmonary-fibrosis-progression' + '/train/'\nmuch_data = []\n\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"viewing the sample of after processing image data, Dont run below cell while forking "},{"metadata":{"trusted":true},"cell_type":"code","source":"for num,patient in enumerate(patients[:2]):\n    if num % 100 == 0:\n        print(num)\n    try:\n        \n        img_data = process_data(patient,data_dir)\n        #print(img_data.shape,label)\n        much_data.append([img_data])\n    except KeyError as e:\n        print('This is unlabeled data!')\n\n#np.save('../input/osic-pulmonary-fibrosis-progression/images/muchdata-{}-{}-{}.npy'.format(64,64,64), much_data)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns=['Weeks','Percent','Age','WHERE','SmokingStatus','Sex']\ntr=tr.drop(columns,axis=1)\ntr.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Concatinating the tabular data with image data using the CtscanDataset function"},{"metadata":{"trusted":true},"cell_type":"code","source":"#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclass CtscanDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True,  meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder        \n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['Patient'] + '.dcm')\n        patient=self.df.iloc[index]['Patient']\n        x = process_data(patient,self.imfolder)\n        \n        try:\n            meta = np.array(self.df.iloc[index,1:].values, dtype=np.float32)\n            meta=torch.tensor(meta,dtype=torch.float32)\n            x=torch.tensor(x,dtype=torch.float32)\n            if self.train:\n                y = self.df.iloc[index]['FVC']\n                y=torch.tensor(y,dtype=torch.float32)\n                return (x, meta), y\n            else:\n                return (x, meta)\n        except:\n            print('error')\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1 = CtscanDataset(df=tr.loc[tr['Patient'].isin(samp)],\n                       imfolder='../input/osic-pulmonary-fibrosis-progression/train/', \n                       train=True,  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1[0][0][0].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Neural network model to work with 3d data and the tabular data combined"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n\n    def __init__(self):\n\n        \n        super(Model, self).__init__()\n\n        self.conv_layer1 = self._make_conv_layer(1, 32)\n        self.conv_layer2 = self._make_conv_layer(32, 64)\n        self.conv_layer4 = self._make_conv_layer(64, 256)\n        self.conv_layer5=nn.Conv3d(256, 256, kernel_size=(1, 3, 3), padding=0)\n        \n        self.fc5 = nn.Linear(4096, 256)\n        self.relu = nn.LeakyReLU()\n        self.batch0=nn.BatchNorm1d(256)\n        self.drop=nn.Dropout(p=0.15)        \n        self.fc6 = nn.Linear(256, 124)\n        self.relu = nn.LeakyReLU()\n        self.batch1=nn.BatchNorm1d(124)\n        \n        self.drop=nn.Dropout(p=0.15)\n        self.fc7 = nn.Linear(124, 128)\n        \n        \n        self.layer1 = nn.Linear(16,128)\n        self.relu1 = nn.LeakyReLU()\n        self.layer2 = nn.Linear(128,128)\n        self.relu2 = nn.LeakyReLU()\n        self.out1 = nn.Linear(256,3)\n        self.relu3 = nn.ReLU()\n        self.out2 = nn.Linear(256,3)\n\n    def _make_conv_layer(self, in_c, out_c):\n        conv_layer = nn.Sequential(\n        nn.Conv3d(in_c, out_c, kernel_size=(2, 3, 3), padding=0),\n        nn.LeakyReLU(),\n        nn.Conv3d(out_c, out_c, kernel_size=(2, 3, 3), padding=1),\n        nn.LeakyReLU(),\n        nn.MaxPool3d((2, 2, 2)),\n        )\n        return conv_layer\n\n    def forward(self, inputs):\n        #print(x.size())\n        x,meta=inputs\n        meta = self.relu1(self.layer1(meta))\n        meta = self.relu2(self.layer2(meta))\n        x = self.conv_layer1(x)\n        #print(x.size())\n        x = self.conv_layer2(x)\n        x = self.conv_layer4(x)\n        #print(x.size())\n        x=self.conv_layer5(x)\n        #print(x.size())\n        x = x.view(x.size(0), -1)\n        x = self.fc5(x)\n        x = self.relu(x)\n        x = self.batch0(x)\n        x = self.drop(x)\n        x = self.fc6(x)\n        x = self.relu(x)\n        x = self.batch1(x)\n        x = self.drop(x)\n        x = self.fc7(x)\n        feat = torch.cat((x, meta), dim=1)\n        o1 = self.out1(feat)\n        o2 = F.relu(self.out2(feat))\n        return o1 + torch.cumsum(o2,dim=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Defining the loss function and training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef score(outputs,target):\n    confidence = outputs[:,2] - outputs[:,0]\n    clip = torch.clamp(confidence,min=70)\n    target=torch.reshape(target,outputs[:,1].shape)\n    delta = torch.abs(outputs[:,1] - target)\n    delta = torch.clamp(delta,max=1000)\n    sqrt_2 = torch.sqrt(torch.tensor([2.])).to(device)\n    metrics = (delta*sqrt_2/clip) + torch.log(clip*sqrt_2)\n    return torch.mean(metrics)\n\ndef qloss(outputs,target):\n    qs = [0.25,0.5,0.75]\n    qs = torch.tensor(qs,dtype=torch.float).to(device)\n    e =  target - outputs\n    e.to(device)\n    v = torch.max(qs*e,(qs-1)*e)\n    v = torch.sum(v,dim=1)\n    return torch.mean(v)\n\ndef loss_fn(outputs,target,l):\n    return l * qloss(outputs,target) + (1- l) * score(outputs,target)\n\ndef train_loop(train_loader,model,loss_fn,device,optimizer,lr_scheduler=None):\n    model.train()\n    losses = list()\n    metrics = list()\n    for i, (data, labels) in enumerate(train_loader):\n        data[0] = data[0].to(device)\n        data[1] = data[1].to(device)\n        labels = labels.to(device)\n\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):           \n            outputs = model(data)                 \n            metric = score(outputs,labels)\n\n            loss = loss_fn(outputs,labels,0.8)\n            metrics.append(metric.cpu().detach().numpy())\n            losses.append(loss.cpu().detach().numpy())\n\n            loss.backward()\n\n            optimizer.step()\n            if lr_scheduler != None:\n                lr_scheduler.step()\n\n    return losses,metrics\n\ndef valid_loop(valid_loader,model,loss_fn,device):\n    model.eval()\n    losses = list()\n    metrics = list()\n    for i, (inputs, labels) in enumerate(valid_loader):\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n\n        outputs = model(inputs)                 \n        metric = score(outputs,labels)\n\n        loss = loss_fn(outputs,labels,0.8)\n        metrics.append(metric.cpu().detach().numpy())\n        losses.append(loss.cpu().detach().numpy())\n\n    return losses,metrics    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size =16\ntrain_1 = CtscanDataset(df=tr.loc[tr['Patient'].isin(samp)][:3],\n                       imfolder='../input/osic-pulmonary-fibrosis-progression/train/', \n                       train=True,  )\ntrain_loader = torch.utils.data.DataLoader(dataset=train_1,\n                                           batch_size=batch_size, shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\nn_epochs = 10\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = Model()\nmodel.to(device)\noptimizer = optim.Adam(model.parameters(),lr=0.1)\nlr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)\n\nfor epoch in range(n_epochs):\n    print(epoch)\n    train_loop(train_loader,model,loss_fn,device,optimizer,lr_scheduler)\n    #evaluate(train_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Credits to the following note books for haleping me finish this:\n- For Model,loss function: https://www.kaggle.com/maunish/osic-super-cool-eda-and-pytorch-baseline\n- For concatinating image and tabular data: https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet\n- For chinking the image data: https://www.kaggle.com/sentdex/first-pass-through-data-w-3d-convnet\n- For preprocessing of the image data: https://www.kaggle.com/gzuidhof/full-preprocessing-tutorial"},{"metadata":{},"cell_type":"markdown","source":"Improvements:\n- Better Image processing\n- Playing with the model\n    "}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}