{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:25:22.272380Z","iopub.status.busy":"2020-09-27T19:25:22.271515Z","iopub.status.idle":"2020-09-27T19:25:25.883728Z","shell.execute_reply":"2020-09-27T19:25:25.882681Z"},"papermill":{"duration":3.668484,"end_time":"2020-09-27T19:25:25.883894","exception":false,"start_time":"2020-09-27T19:25:22.215410","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport sys\nimport random\nimport torch\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport math\nimport plotly\nimport plotly_express as px\nimport seaborn as sns\nimport itertools\nfrom functools import partial\nimport tqdm.notebook as tqdm\nfrom scipy.stats import kurtosis, skew\nimport time\nimport pydicom\nimport scipy\nimport scipy.ndimage as ndimage\nfrom skimage import measure, morphology, segmentation\nfrom scipy.ndimage.interpolation import zoom\nfrom PIL import Image \n\n# from pytorch_tabnet.tab_network import TabNet\n\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n\n%load_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:25:25.948734Z","iopub.status.busy":"2020-09-27T19:25:25.947776Z","iopub.status.idle":"2020-09-27T19:25:25.996626Z","shell.execute_reply":"2020-09-27T19:25:25.996053Z"},"papermill":{"duration":0.082699,"end_time":"2020-09-27T19:25:25.996735","exception":false,"start_time":"2020-09-27T19:25:25.914036","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorchtabnet/tabnet-develop')\nfrom pytorch_tabnet.tab_network import TabNet","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:25:26.065675Z","iopub.status.busy":"2020-09-27T19:25:26.064728Z","iopub.status.idle":"2020-09-27T19:26:25.847581Z","shell.execute_reply":"2020-09-27T19:26:25.846937Z"},"papermill":{"duration":59.820586,"end_time":"2020-09-27T19:26:25.847724","exception":false,"start_time":"2020-09-27T19:25:26.027138","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install ../input/pytorchlightning/tensorboard-2.2.0-py3-none-any.whl -q\n!pip install ../input/pytorchlightning/pytorch_lightning-0.9.0-py3-none-any.whl -q","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:25.932483Z","iopub.status.busy":"2020-09-27T19:26:25.927221Z","iopub.status.idle":"2020-09-27T19:26:25.985597Z","shell.execute_reply":"2020-09-27T19:26:25.986138Z"},"papermill":{"duration":0.104953,"end_time":"2020-09-27T19:26:25.986299","exception":false,"start_time":"2020-09-27T19:26:25.881346","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"## Code from https://www.kaggle.com/aadhavvignesh/lung-segmentation-by-marker-controlled-watershed \n\ndef generate_markers(image):\n    \"\"\"\n    Generates markers for a given image.\n    \n    Parameters: image\n    \n    Returns: Internal Marker, External Marker, Watershed Marker\n    \"\"\"\n    \n    #Creation of the internal Marker\n    marker_internal = image < -400\n    marker_internal = segmentation.clear_border(marker_internal)\n    marker_internal_labels = measure.label(marker_internal)\n    \n    areas = [r.area for r in measure.regionprops(marker_internal_labels)]\n    areas.sort()\n    \n    if len(areas) > 2:\n        for region in measure.regionprops(marker_internal_labels):\n            if region.area < areas[-2]:\n                for coordinates in region.coords:                \n                       marker_internal_labels[coordinates[0], coordinates[1]] = 0\n    \n    marker_internal = marker_internal_labels > 0\n    \n    # Creation of the External Marker\n    external_a = ndimage.binary_dilation(marker_internal, iterations=10)\n    external_b = ndimage.binary_dilation(marker_internal, iterations=55)\n    marker_external = external_b ^ external_a\n    \n    # Creation of the Watershed Marker\n    marker_watershed = np.zeros(image.shape, dtype=np.int)\n    marker_watershed += marker_internal * 255\n    marker_watershed += marker_external * 128\n    \n    return marker_internal, marker_external, marker_watershed\n\n\ndef seperate_lungs(image,iterations = 1):\n    \"\"\"\n    Segments lungs using various techniques.\n    \n    Parameters: image (Scan image), iterations (more iterations, more accurate mask)\n    \n    Returns: \n        - Segmented Lung\n        - Lung Filter\n        - Outline Lung\n        - Watershed Lung\n        - Sobel Gradient\n    \"\"\"\n    \n    # Store the start time\n    # start = time.time()\n    marker_internal, marker_external, marker_watershed = generate_markers(image)\n    \n    \n    '''\n    Creation of Sobel Gradient\n    '''\n    \n    # Sobel-Gradient\n    sobel_filtered_dx = ndimage.sobel(image, 1)\n    sobel_filtered_dy = ndimage.sobel(image, 0)\n    sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)\n    sobel_gradient *= 255.0 / np.max(sobel_gradient)\n    \n    \n    '''\n    Using the watershed algorithm\n    \n    \n    We pass the image convoluted by sobel operator and the watershed marker\n    to morphology.watershed and get a matrix matrix labeled using the \n    watershed segmentation algorithm.\n    '''\n    watershed = morphology.watershed(sobel_gradient, marker_watershed)\n    \n    '''\n    Reducing the image to outlines after Watershed algorithm\n    '''\n    outline = ndimage.morphological_gradient(watershed, size=(3,3))\n    outline = outline.astype(bool)\n    \n    \n    '''\n    Black Top-hat Morphology:\n    \n    The black top hat of an image is defined as its morphological closing\n    minus the original image. This operation returns the dark spots of the\n    image that are smaller than the structuring element. Note that dark \n    spots in the original image are bright spots after the black top hat.\n    '''\n    \n    # Structuring element used for the filter\n    blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],\n                       [0, 1, 1, 1, 1, 1, 0],\n                       [1, 1, 1, 1, 1, 1, 1],\n                       [1, 1, 1, 1, 1, 1, 1],\n                       [1, 1, 1, 1, 1, 1, 1],\n                       [0, 1, 1, 1, 1, 1, 0],\n                       [0, 0, 1, 1, 1, 0, 0]]\n    \n    blackhat_struct = ndimage.iterate_structure(blackhat_struct, iterations)\n    \n    # Perform Black Top-hat filter\n    outline += ndimage.black_tophat(outline, structure=blackhat_struct)\n    \n    '''\n    Generate lung filter using internal marker and outline.\n    '''\n    lungfilter = np.bitwise_or(marker_internal, outline)\n    lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones((5,5)), iterations=3)\n    \n    '''\n    Segment lung using lungfilter and the image.\n    '''\n    segmented = np.where(lungfilter, image, -1000)\n    \n    #return segmented, lungfilter, outline, watershed, sobel_gradient\n    return segmented","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:26.083872Z","iopub.status.busy":"2020-09-27T19:26:26.069884Z","iopub.status.idle":"2020-09-27T19:26:26.137949Z","shell.execute_reply":"2020-09-27T19:26:26.137352Z"},"papermill":{"duration":0.119776,"end_time":"2020-09-27T19:26:26.138063","exception":false,"start_time":"2020-09-27T19:26:26.018287","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def load_scan(path):\n    paths = os.listdir(path)\n    paths = sorted(paths, key=lambda x: int(str(x).split('/')[-1].split('.')[0]))\n    slices = [pydicom.read_file(path + '/' + s) for s in paths]\n    print(len(slices), 'slices')\n    try:\n        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    except:\n        pass\n    try:\n        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n    except:\n        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n    for s in slices:\n        s.SliceThickness = slice_thickness\n    return slices\n\ndef get_pixels_hu(slices):\n    image = np.stack([s.pixel_array for s in slices])\n    # Convert to int16 (from sometimes int16), \n    # should be possible as values should always be low enough (<32k)\n    image = image.astype(np.int16)\n\n    # Set outside-of-scan pixels to 0\n    # The intercept is usually -1024, so air is approximately 0\n    \n    \n    # Convert to Hounsfield units (HU)\n    for slice_number in range(len(slices)):\n        \n        intercept = slices[slice_number].RescaleIntercept\n        slope = slices[slice_number].RescaleSlope\n        \n        if slope != 1:\n            image[slice_number] = slope * image[slice_number].astype(np.float64)\n            image[slice_number] = image[slice_number].astype(np.int16)\n            \n        image[slice_number] += np.int16(intercept)\n    image[image <= -1900] = -1000\n    return np.array(image, dtype=np.int16)\n\ndef resample(image, scan, new_spacing=[1,1,1]):\n    st = time.time()\n    slice_thickness = scan[0].SliceThickness\n    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)\n    resize_factor = spacing / new_spacing\n    new_real_shape = image.shape * resize_factor\n    new_shape = np.round(new_real_shape)\n    real_resize_factor = new_shape / image.shape\n    new_spacing = spacing / real_resize_factor\n    print('resample factor', time.time() - st, real_resize_factor)\n    image = scipy.ndimage.zoom(image, real_resize_factor, mode='nearest')\n    print('resample time', time.time() - st)\n    return image, new_spacing\n\ndef torch_resample(image, scan, new_spacing=[1,1,1]):\n    st = time.time()\n    slice_thickness = scan[0].SliceThickness\n    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)\n    resize_factor = spacing / new_spacing\n    new_real_shape = image.shape * resize_factor\n    new_shape = np.round(new_real_shape)\n    real_resize_factor = new_shape / image.shape\n    new_spacing = spacing / real_resize_factor\n    #print('resample factor', time.time() - st, real_resize_factor)\n    image = torch.tensor(image).unsqueeze(0).unsqueeze(0)\n    #print(real_resize_factor)\n    #size = [int(x*y) for x, y in zip(real_resize_factor, image.shape[-3:])]\n    #print(image.shape)\n    #print('using resize')\n    image = torch.nn.functional.interpolate(image, scale_factor=tuple(real_resize_factor), mode='nearest')\n    image = image.squeeze(0).squeeze(0).numpy()\n    #print(image.shape)\n    #print('resample time', time.time() - st)\n    return image, new_spacing\n\n\n\n\ndef get_3d_resampled_array(patient_path):\n    start_time = time.time()\n    patient_slices = load_scan(str(patient_path))\n    patient_slices_hu = get_pixels_hu(patient_slices)\n    print('HU loaded', time.time() - start_time)\n    print(patient_slices_hu.shape)\n    #lungmask_3d = np.apply_over_axes(seperate_lungs, patient_slices_hu, 0)\n    idx = np.ndindex(patient_slices_hu.shape[0])\n    patient_slices_hu_masked = np.zeros(patient_slices_hu.shape)\n    for i in idx:\n        patient_slices_hu_masked[i] = seperate_lungs(patient_slices_hu[i])\n        #patient_slices_hu_masked[i, :, :] = np.where(lungmask, patient_slices_hu[i, :, :], -1000)\n    #patient_slices_hu_masked = np.where(lungmask_3d, patient_slices_hu, -1000)\n    \n    print('mask generated', time.time() - start_time)\n    resampled_array, spacing = torch_resample(patient_slices_hu_masked, patient_slices, [1,1,1])\n    print('after resample', time.time() - start_time)\n    return resampled_array, spacing\n\n\ndef get_features_from_3d_array(resampled_array, spacing):\n    features = {}\n    \n    # volume of lungs\n    cube_volume = spacing[0] * spacing[1] * spacing[2]\n    total_lung_volume = (resampled_array[resampled_array > -900].shape[0] * cube_volume)\n    lung_volume_in_liters = total_lung_volume / (1000*1000)\n    features['lung_volume_in_liters'] = lung_volume_in_liters\n    \n    #HU unit binning\n    bins_threshold = (resampled_array <= 300) & (resampled_array >= -900)\n    total_hu_units_bin = resampled_array[bins_threshold].flatten().shape[0]\n    bin_values, bins = np.histogram(resampled_array[bins_threshold].flatten(), bins=range(-900, 400, 100))\n    features['total_hu_units_bin'] = total_hu_units_bin\n    for i, _bin in enumerate(bins[:-1]):\n        features[f'bin_{_bin}'] = bin_values[i] / total_hu_units_bin\n    \n    #mean, skew, kurtosis\n    lung_threshold = (resampled_array <= -320) & (resampled_array >= -900)\n    histogram_values, _ = np.histogram(resampled_array[lung_threshold].flatten(), bins=100)\n    features['lung_mean_hu'] = np.mean(resampled_array[lung_threshold].flatten())\n    features['lung_skew'] = skew(histogram_values)\n    features['lung_kurtosis'] = kurtosis(histogram_values)\n    \n    #height_of_lung\n    n_lung_pixels = lung_threshold.sum(axis=1).sum(axis=1)\n    height_start = np.argwhere(n_lung_pixels > 1000).min()\n    height_end = np.argwhere(n_lung_pixels > 1000).max()\n    features['height_of_lung_cm'] = (height_end - height_start)/10\n    \n    return features","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:26.207430Z","iopub.status.busy":"2020-09-27T19:26:26.206462Z","iopub.status.idle":"2020-09-27T19:26:26.244823Z","shell.execute_reply":"2020-09-27T19:26:26.243990Z"},"papermill":{"duration":0.074999,"end_time":"2020-09-27T19:26:26.244976","exception":false,"start_time":"2020-09-27T19:26:26.169977","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"data_dir = Path('../input/osic-pulmonary-fibrosis-progression')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:26.318462Z","iopub.status.busy":"2020-09-27T19:26:26.317450Z","iopub.status.idle":"2020-09-27T19:26:26.357151Z","shell.execute_reply":"2020-09-27T19:26:26.356528Z"},"papermill":{"duration":0.079305,"end_time":"2020-09-27T19:26:26.357273","exception":false,"start_time":"2020-09-27T19:26:26.277968","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:26.429310Z","iopub.status.busy":"2020-09-27T19:26:26.428136Z","iopub.status.idle":"2020-09-27T19:26:26.471243Z","shell.execute_reply":"2020-09-27T19:26:26.470608Z"},"papermill":{"duration":0.080889,"end_time":"2020-09-27T19:26:26.471373","exception":false,"start_time":"2020-09-27T19:26:26.390484","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"seed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_submission(submission):\n    submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\n    submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\n    return submission","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:26.662994Z","iopub.status.busy":"2020-09-27T19:26:26.661981Z","iopub.status.idle":"2020-09-27T19:26:26.732618Z","shell.execute_reply":"2020-09-27T19:26:26.731032Z"},"papermill":{"duration":0.109545,"end_time":"2020-09-27T19:26:26.732748","exception":false,"start_time":"2020-09-27T19:26:26.623203","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(data_dir/'train.csv')\ntest = pd.read_csv(data_dir/'test.csv')\n# dicom_meta = pd.read_pickle(data_dir/'train_dicom_df')/\nsubmission = process_submission(pd.read_csv(data_dir/'sample_submission.csv'))\nimage_feature_df = pd.read_csv('../input/lung-image-features/patient_feature2_df.csv')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:27.071013Z","iopub.status.busy":"2020-09-27T19:26:27.070021Z","iopub.status.idle":"2020-09-27T19:26:27.113880Z","shell.execute_reply":"2020-09-27T19:26:27.113264Z"},"papermill":{"duration":0.08441,"end_time":"2020-09-27T19:26:27.113994","exception":false,"start_time":"2020-09-27T19:26:27.029584","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"image_feature_df =image_feature_df.drop(['Unnamed: 0'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:26:27.189701Z","iopub.status.busy":"2020-09-27T19:26:27.188705Z","iopub.status.idle":"2020-09-27T19:26:27.250843Z","shell.execute_reply":"2020-09-27T19:26:27.251402Z"},"papermill":{"duration":0.10231,"end_time":"2020-09-27T19:26:27.251543","exception":false,"start_time":"2020-09-27T19:26:27.149233","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"image_feature_df.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.041986,"end_time":"2020-09-27T19:32:09.492065","exception":false,"start_time":"2020-09-27T19:32:09.450079","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset building"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:09.582016Z","iopub.status.busy":"2020-09-27T19:32:09.581167Z","iopub.status.idle":"2020-09-27T19:32:09.615748Z","shell.execute_reply":"2020-09-27T19:32:09.615284Z"},"papermill":{"duration":0.080975,"end_time":"2020-09-27T19:32:09.615856","exception":false,"start_time":"2020-09-27T19:32:09.534881","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Validataion split"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, train_test_split, KFold\nfrom sklearn.preprocessing import LabelEncoder\n\ndef frequency_binning(x, nbin):\n    nlen = len(x)\n    return np.unique(np.interp(np.linspace(0, nlen, nbin + 1),\n                     np.arange(nlen),\n                     np.sort(x)))\n\ndef replace_with_frequency_bins(arr, nbins=5):\n    bins = frequency_binning(arr, nbins)\n    out = pd.cut(arr, bins)\n    return [inter.left if (type(inter) is not float) else inter for inter in out]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['expected_fvc'] = np.round((100 * train['FVC']) / train['Percent'], 2)\ntest['expected_fvc'] = np.round((100 * test['FVC']) / test['Percent'], 2)\npatient_df = train[['Patient', 'Weeks', 'Age', 'Sex', 'expected_fvc', 'SmokingStatus']]\npatient_df['num_weeks'] = train[['Patient', 'Weeks']].groupby('Patient').transform('count')\npatient_df['min_week'] = train[['Patient', 'Weeks']].groupby('Patient').transform('min')\n_le = LabelEncoder()\npatient_df['SmokingStatus'] = _le.fit_transform(train['SmokingStatus'])\npatient_df['Sex'] = (patient_df['Sex'] == 'Male').astype(int)\n\npatient_df['Age'] = replace_with_frequency_bins(patient_df['Age'], 5)\npatient_df['num_weeks'] = replace_with_frequency_bins(patient_df['num_weeks'], 5)\npatient_df['min_week'] = replace_with_frequency_bins(patient_df['min_week'], 5)\npatient_df['expected_fvc'] = pd.Series(replace_with_frequency_bins(patient_df['expected_fvc'], 6)).astype(str)\n\npatient_df = patient_df.drop('Weeks',axis=1).drop_duplicates()\npatient_df.index = list(range(len(patient_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_df = patient_df.merge(image_feature_df, left_on='Patient', right_on='patient_id').drop('patient_id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:09.725627Z","iopub.status.busy":"2020-09-27T19:32:09.709786Z","iopub.status.idle":"2020-09-27T19:32:09.809386Z","shell.execute_reply":"2020-09-27T19:32:09.808862Z"},"papermill":{"duration":0.151491,"end_time":"2020-09-27T19:32:09.809483","exception":false,"start_time":"2020-09-27T19:32:09.657992","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class LungDataset(Dataset):\n    def __init__(self, mode, data, image_features, preprocessing_params=None, expand=True):\n        data = data.drop_duplicates(subset=['Patient', 'Weeks'])\n        self.inference_data = None\n        self.ohe = False\n        self.preprocessing_params = preprocessing_params if preprocessing_params is not None else {\n            'min_week': -12,\n            'max_week': 133\n        }\n        self.expand = expand\n        self.raw_data = self.process_data(data.copy(), image_features, mode, self.preprocessing_params)\n        self.data = self.raw_data[self.features]\n        self.target = self.raw_data['FVC'] if mode != 'test' else None\n        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n        self.inference = False\n        self.pass_y = True if mode != 'test' else False\n        \n    def expand_train_data(self, train):\n        train_expanded = pd.DataFrame()\n        for patient, patient_df in train.groupby('Patient'):\n            patient_expanded = pd.DataFrame()\n            expected_fvc = np.mean((100 * patient_df['FVC']) / patient_df['Percent'])\n            for week in patient_df['Weeks'].sort_values():\n                week_df = patient_df.copy()\n                week_df['weeks_from_first_visit'] = week_df['Weeks'] - week\n                week_df['first_test_fvc'] = patient_df.loc[patient_df['Weeks'] == week, 'FVC'].mean()\n                week_df['first_test_week'] = week\n                week_df['predict_week'] = patient_df['Weeks']\n                week_df.drop(['Percent', 'Weeks'], axis=1, inplace=True)\n                patient_expanded = pd.concat([patient_expanded, week_df], ignore_index=True)\n            patient_expanded['expected_fvc'] = expected_fvc\n            train_expanded = pd.concat([train_expanded, patient_expanded], ignore_index=True)\n            train_expanded = train_expanded[(train_expanded['weeks_from_first_visit'] >= -12) & (train_expanded['weeks_from_first_visit'] <= 133)]\n        return train_expanded\n\n    def process_data(self, data, img_data, mode, params):\n        if mode == 'train' and self.expand:\n            data = self.expand_train_data(data)\n        else:\n            # expected_FVC\n            data['expected_fvc'] = (100 * data['FVC']) / data['Percent']\n            data['first_test_week'] = data[['Patient', 'Weeks']].groupby('Patient').transform('min')\n            data['weeks_from_first_visit'] = data['Weeks'] - data['first_test_week']\n            data['predict_week'] = data['Weeks']\n            min_fvc_df = data.sort_values('first_test_week').groupby('Patient').head(1)[['Patient', 'FVC']]\n            min_fvc_df.columns = ['Patient', 'first_test_fvc']\n            data = data.merge(min_fvc_df, on='Patient', how='left')\n        data['percent'] = data['first_test_fvc'] / data['expected_fvc']\n            \n        \n        # age\n        min_age = params.get('min_age', data['Age'].min())\n        max_age = params.get('max_age', data['Age'].max())\n        data['age'] = (data['Age'] - data['Age'].min()) / (data['Age'].max() - data['Age'].min())\n        \n        # OHE\n        if self.ohe:\n            sex_df = pd.get_dummies(data['Sex']).reset_index(drop=True)\n            sex_columns = ['Male', 'Female']\n            for col in sex_columns:\n                if col not in sex_df:\n                    sex_df[col] = 0\n            smoke_df = pd.get_dummies(data['SmokingStatus']).reset_index(drop=True)\n            smoke_columns = ['Currently smokes','Ex-smoker','Never smoked']\n            for col in smoke_columns:\n                if col not in smoke_df.columns:\n                    smoke_df[col] = 0\n                    \n        else:\n            sex_map = {'Male': 0, 'Female': 1}\n            smoke_map = {'Ex-smoker': 0, 'Never smoked': 1, 'Currently smokes': 2}\n            data['sex'] = data['Sex'].map(sex_map)\n            data['smoke'] = data['SmokingStatus'].map(smoke_map)\n            \n        # base week\n        min_week = params.get('min_week')\n        max_week = params.get('max_week')\n        data['first_test_week'] = (data['first_test_week'] - min_week) / (max_week - min_week)\n        data['weeks_from_first_visit'] = (data['weeks_from_first_visit'] - min_week) / (max_week - min_week)\n        data['predict_week'] = (data['predict_week'] - min_week) / (max_week - min_week)\n        \n        # Opting min_FVC value\n        min_FVC = params.get('min_fvc', min(data['expected_fvc'].min(), data['FVC'].min()))\n        max_FVC = params.get('max_fvc', max(data['expected_fvc'].max(), data['FVC'].max()))\n        data['expected_fvc'] = (data['expected_fvc'] - min_FVC) / (max_FVC - min_FVC)\n        data['first_test_fvc'] = (data['first_test_fvc'] - min_FVC) /(max_FVC - min_FVC)\n        data['FVC'] = (data['FVC'] - min_FVC) /(max_FVC - min_FVC)\n    \n        #print(data.shape, sex_df.shape, smoke_df.shape)\n        data.reset_index(drop=True, inplace=True)\n        if self.ohe:\n            data = pd.concat([data, sex_df, smoke_df], axis=1)\n        #print(data.shape)\n        # update params\n        self.preprocessing_params = {\n            'min_age': min_age,\n            'max_age': max_age,\n            'min_week': min_week,\n            'max_week': max_week,\n            'min_fvc': min_FVC,\n            'max_fvc': max_FVC\n        }\n        \n        self.features = ['first_test_fvc',\n                         'age', 'predict_week', 'percent', 'expected_fvc', 'weeks_from_first_visit'] #'first_test_week']\n        \n        #Image features\n        if img_data is not None:\n            # lung_volume\n            missing_img_data = img_data[img_data['missing'] == 1].copy()\n            img_data = img_data[img_data['missing'] == 0].copy()\n            min_lung_vol = params.get('min_lung_vol', img_data['lung_volume_in_liters'].min())\n            max_lung_vol = params.get('max_lung_vol', img_data['lung_volume_in_liters'].max())\n            img_data['lung_volume'] = (img_data['lung_volume_in_liters'] - min_lung_vol) / (max_lung_vol - min_lung_vol)\n            \n            # lung height\n            min_lung_height = params.get('min_lung_height', img_data['height_of_lung_cm'].min())\n            max_lung_height = params.get('max_lung_height', img_data['height_of_lung_cm'].max())\n            img_data['lung_height'] = (img_data['height_of_lung_cm'] - min_lung_height) / (max_lung_height - min_lung_height)\n            \n            #mean HU value\n            min_hu_mean = params.get('min_hu_mean', img_data['lung_mean_hu'].min())\n            max_hu_mean = params.get('max_hu_mean', img_data['lung_mean_hu'].max())\n            img_data['hu_mean'] = (img_data['lung_mean_hu'] - min_hu_mean) / (max_hu_mean - min_hu_mean)\n            \n            #skew\n            min_hu_skew = params.get('min_hu_skew', img_data['lung_skew'].min())\n            max_hu_skew = params.get('max_hu_skew', img_data['lung_skew'].max())\n            img_data['hu_skew'] = (img_data['lung_skew'] - min_hu_skew) / (max_hu_skew - min_hu_skew)\n        \n            #kurtosis\n            min_hu_kurtosis = params.get('min_hu_kurtosis', img_data['lung_kurtosis'].min())\n            max_hu_kurtosis = params.get('max_hu_kurtosis', img_data['lung_kurtosis'].max())\n            img_data['hu_kurtosis'] = (img_data['lung_kurtosis'] - min_hu_kurtosis) / (max_hu_kurtosis - min_hu_kurtosis)\n            \n            img_features = ['lung_volume', 'lung_height', 'hu_mean', 'hu_skew', 'hu_kurtosis', 'missing']\n            self.features += img_features\n            self.preprocessing_params.update({\n                'min_lung_vol': min_lung_vol,\n                'max_lung_vol': max_lung_vol,\n                'min_lung_height': min_lung_height,\n                'max_lung_height': max_lung_height,\n                'min_hu_mean': min_hu_mean,\n                'max_hu_mean': max_hu_mean,\n                'min_hu_skew': min_hu_skew,\n                'max_hu_skew': max_hu_skew,\n                'min_hu_kurtosis': min_hu_kurtosis,\n                'max_hu_kurtosis': max_hu_kurtosis,\n            })\n            bin_features = [x for x in img_data.columns if 'bin_' in x]\n            self.features += bin_features\n            img_feature_df = img_data[[*img_features, *bin_features, 'patient_id']]\n            for col in img_feature_df.columns:\n                if col not in missing_img_data:\n                    missing_img_data[col] = 0\n            img_feature_df = pd.concat([img_feature_df, missing_img_data], ignore_index=True)\n            img_feature_df = img_feature_df.fillna(0)\n            data = data.merge(img_feature_df, left_on='Patient', right_on='patient_id')\n        \n        if self.ohe:\n            self.features += [*sex_df.columns, *smoke_df.columns]\n        else:\n            self.features += ['sex', 'smoke']\n        data.index = list(range(len(data)))\n        \n        # construct inference data\n        if mode != 'train':\n            inference_df = pd.DataFrame(\n                itertools.product(data['Patient'].unique(), range(-12, 134)), columns=['Patient', 'Weeks']\n            )\n            min_week_df = data[data['weeks_from_first_visit'] == 12 / (133 + 12)].drop('Weeks', axis=1)\n            inference_df = inference_df.merge(min_week_df, on='Patient', how='left')\n            first_test_week = (inference_df['first_test_week'] * (max_week - min_week)) + min_week\n            inference_df['weeks_from_first_visit'] = inference_df['Weeks'] - first_test_week\n            inference_df['predict_week'] = inference_df['Weeks']\n            inference_df['weeks_from_first_visit'] = (inference_df['weeks_from_first_visit'] - min_week) / (max_week - min_week)\n            inference_df['predict_week'] = (inference_df['predict_week'] - min_week) / (max_week - min_week)\n            self.inference_data = inference_df.reset_index(drop=True)\n            \n        return data\n    \n    def get_params(self):\n        return self.preprocessing_params\n    \n    def get_inference_df(self, last_3_visits=False):\n        return self.inference_data\n    \n    def set_inference(self, value):\n        self.inference = value\n        \n    def get_y(self, value):\n        self.pass_y = value\n    \n    def __len__(self):\n        if not self.inference:\n            return len(self.data)\n        return len(self.inference_data)\n    \n    def __getitem__(self, idx):\n        if not self.inference:\n            data = self.data\n        else:\n            data = self.inference_data[self.features]\n        if (self.target is None) or self.inference or (not self.pass_y):\n            return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device)\n        return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device), torch.tensor(self.target[idx], dtype=torch.float32).to(self.device)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:09.900225Z","iopub.status.busy":"2020-09-27T19:32:09.899304Z","iopub.status.idle":"2020-09-27T19:32:10.369188Z","shell.execute_reply":"2020-09-27T19:32:10.368098Z"},"papermill":{"duration":0.516354,"end_time":"2020-09-27T19:32:10.369324","exception":false,"start_time":"2020-09-27T19:32:09.852970","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_dataset = LungDataset('train', train, image_feature_df, expand=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:10.462432Z","iopub.status.busy":"2020-09-27T19:32:10.461479Z","iopub.status.idle":"2020-09-27T19:32:10.500926Z","shell.execute_reply":"2020-09-27T19:32:10.500406Z"},"papermill":{"duration":0.088158,"end_time":"2020-09-27T19:32:10.501032","exception":false,"start_time":"2020-09-27T19:32:10.412874","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_dataset.get_params()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_splits(stratify_df, train_df, image_df, kfolds, stratify_col=None, expand=True):\n    splits = kfolds.split(stratify_df) if not stratify_col else kfolds.split(stratify_df, stratify_df[stratify_col])\n    for train_index, valid_index in splits:\n        train_patients = patient_df.loc[train_index, 'Patient']\n        valid_patients = patient_df.loc[valid_index, 'Patient']\n        batch_train_df = train_df[train_df['Patient'].isin(train_patients)]\n        batch_valid_df = train_df[train_df['Patient'].isin(valid_patients)]\n        batch_train_image_df = image_df[image_df['patient_id'].isin(train_patients)]\n        batch_valid_image_df = image_df[image_df['patient_id'].isin(valid_patients)]\n        train_dataset = LungDataset('train', batch_train_df, batch_train_image_df, expand=expand)\n        params = train_dataset.get_params()\n        valid_dataset = LungDataset('valid', batch_valid_df, batch_valid_image_df, params)\n        yield train_dataset, valid_dataset","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.045832,"end_time":"2020-09-27T19:32:10.592796","exception":false,"start_time":"2020-09-27T19:32:10.546964","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Quantile Regression\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_lightning.callbacks import Callback\n\n\nclass CVResultTracker(Callback):\n    def __init__(self, result_file, cv_result_file):\n        super().__init__()\n        self.result_file = result_file\n        self_cv_result_file = cv_result_file\n        \n    def write_file(self, path, df):\n        if path.is_file():\n            existing_df = pd.read_csv(path)\n            df = pd.concat([existing_df, df], ignore_index=True)\n        df.to_csv(path, index=False)\n        return df\n        \n    def teardown(self, trainer, pl_module, stage):\n        logger = None\n        for logr in trainer.logger:\n            logger = logr if 'CSVLogger' in str(logr.__class__) else None\n        if logger:\n            metrics_df = pd.read_csv(f'./{logger.save_dir}/{logger.name}/{logger.version}/metrics.csv')\n            val_df = metrics_df.dropna(subset=['val_loss'])\n            val_df['name'] = logger.name + '_' + logger.version\n            val_df['model_name'] = logger.name.split('fold')[0]\n            val_df_last = val_df[val_df['epoch'] == val_df['epoch'].max()]\n            val_df_best = val_df[val_df['val_nll'] == val_df['val_nll'].min()]\n            val_df_last = val_df_last.dropna(axis=1, how='all')\n            val_df_best = val_df_best.dropna(axis=1, how='all')\n            fold_results_best_path = Path(f'./{logger.save_dir}/fold_results_best.csv')\n            fold_results_last_path = Path(f'./{logger.save_dir}/fold_results_last.csv')\n            model_results_best_path = Path(f'./{logger.save_dir}/model_results_best.csv')\n            model_results_last_path = Path(f'./{logger.save_dir}/model_results_last.csv')\n            fold_results_best_df = self.write_file(fold_results_best_path, val_df_best)\n            fold_results_last_df = self.write_file(fold_results_last_path, val_df_last)\n            \n            model_best_df = fold_results_best_df.drop('name', axis=1)\n            model_last_df = fold_results_last_df.drop('name', axis=1)\n            \n            model_best_df = model_best_df.groupby('model_name', as_index=False).mean()\n            model_last_df = model_last_df.groupby('model_name', as_index=False).mean()\n            \n            model_best_df.to_csv(model_results_best_path, index=False)\n            model_last_df.to_csv(model_results_last_path, index=False)\n            \n            ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:10.701294Z","iopub.status.busy":"2020-09-27T19:32:10.700289Z","iopub.status.idle":"2020-09-27T19:32:16.465413Z","shell.execute_reply":"2020-09-27T19:32:16.464867Z"},"papermill":{"duration":5.826487,"end_time":"2020-09-27T19:32:16.465529","exception":false,"start_time":"2020-09-27T19:32:10.639042","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.metrics.metric import TensorMetric\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.callbacks import LearningRateLogger\n\n\nfrom pytorch_lightning import loggers\n\n\n\nclass NLL(TensorMetric):\n    name = 'NLL'\n    def __init__(self, min_fvc, max_fvc):\n        super().__init__(name='nll')\n        self.min_fvc = min_fvc\n        self.max_fvc = max_fvc\n        \n    def forward(self, preds, target):\n        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc\n        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc\n        sigma = preds[:, 2] - preds[:, 0]\n        fvc_pred = preds[:, 1]\n\n        #sigma_clip = sigma + C1\n        sigma_clip = torch.max(sigma, torch.tensor(70.).to(self.device))\n        delta = torch.abs(target - fvc_pred)\n        delta = torch.min(delta, torch.tensor(1000.).to(self.device))\n        sq2 = torch.sqrt(torch.tensor(2.).to(device))\n        metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)\n        # print(torch.mean(metric))\n        return torch.mean(metric)\n    \nclass MAE(TensorMetric):\n    name = 'MAE'\n    def __init__(self, min_fvc, max_fvc):\n        super().__init__(name='MAE')\n        self.min_fvc = min_fvc\n        self.max_fvc = max_fvc\n        \n    def forward(self, preds, target):\n        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc\n        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc\n        return torch.mean(torch.abs(target - preds[:, 1]))\n    \n\nclass LQuantModel(pl.LightningModule):\n    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,\n                 out_quantiles=3):\n        super(LQuantModel, self).__init__()\n        self.learning_rate = learning_rate\n        self.total_steps = total_steps\n        self.fc1 = nn.Linear(in_tabular_features, 512)\n        self.fc2 = nn.Linear(512, 256)\n        self.fc3 = nn.Linear(256, out_quantiles)\n        self.fc4 = nn.Linear(256, out_quantiles)\n        self.train_params = train_params\n        self.save_hyperparameters()\n        self.metrics = [\n            NLL(\n                train_params.get('min_fvc'),\n                train_params.get('max_fvc'),\n            ),\n            MAE(\n                train_params.get('min_fvc'),\n                train_params.get('max_fvc'),\n            )\n        ]\n        self.eval_tracker = {}\n    \n        \n    def forward(self, x):\n        x = F.relu(self.fc1(x))   \n        x = F.relu(self.fc2(x))\n        x1 = self.fc3(x)\n        x2 = F.relu(self.fc4(x))\n        preds = x1 + torch.cumsum(x2, axis=1)\n        return preds\n    \n    def configure_optimizers(self):\n        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)\n        scheduler = optim.lr_scheduler.OneCycleLR(\n            optimizer,\n            max_lr=self.learning_rate,\n            total_steps=self.total_steps\n        )\n        return [optimizer], [{\n            'scheduler': scheduler,\n            'interval': 'step',\n            'frequency': 1\n        }]\n        \n    def set_loss_fn(self, loss_fn):\n        self.loss_fn = loss_fn\n        \n        \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.loss_fn(y_hat, y)\n        result = pl.TrainResult(minimize=loss)\n        result.log('train_loss', loss, prog_bar=True)\n        for metric in self.metrics:\n            metric.to(self.device)\n            metric_value = metric(y_hat, y)\n            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)\n        return result\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.loss_fn(y_hat, y)\n        result = pl.EvalResult(checkpoint_on=loss)\n        result.log('val_loss', loss, on_epoch=True, on_step=False)\n        for metric in self.metrics:\n            metric.to(self.device)\n            metric_value = metric(y_hat, y)\n            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)\n        return result\n    \n#     def validation_epoch_end(self, validation_results):\n#         self.eval_tracker = {k: v[-1] for k, v in validation_results.items() if k[0:4] == 'val_'}\n#         return validation_results\n    \n    def test_step(self, batch, batch_idx):\n        y_hat = self(batch)\n        result = pl.EvalResult()\n        result.predictions = y_hat\n        return result\n    \n    \n    \nclass LTabnet(pl.LightningModule):\n    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,\n                 out_quantiles=3):\n        super(LTabnet, self).__init__()\n        self.learning_rate = learning_rate\n        self.total_steps = total_steps\n        self.tabnet = TabNet(\n            input_dim=26,\n            output_dim=3,\n            n_a=64,\n            n_d=64,\n            cat_idxs=[24,25],\n            cat_dims=[2,3],\n            cat_emb_dim=[4,4],\n        ).to(self.device)\n        self.train_params = train_params\n        self.save_hyperparameters()\n        self.metrics = [\n            NLL(\n                train_params.get('min_fvc'),\n                train_params.get('max_fvc'),\n            ),\n            MAE(\n                train_params.get('min_fvc'),\n                train_params.get('max_fvc'),\n            ),\n        ]\n    \n        \n    def forward(self, x):\n        preds, _ = self.tabnet(x)\n        return preds\n    \n    def configure_optimizers(self):\n        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)\n        scheduler = optim.lr_scheduler.OneCycleLR(\n            optimizer,\n            max_lr=self.learning_rate,\n            total_steps=self.total_steps\n        )\n        return [optimizer], [{\n            'scheduler': scheduler,\n            'interval': 'step',\n            'frequency': 1\n        }]\n        \n    def set_loss_fn(self, loss_fn):\n        self.loss_fn = loss_fn\n        \n        \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.loss_fn(y_hat, y)\n        result = pl.TrainResult(minimize=loss)\n        result.log('train_loss', loss, prog_bar=True)\n        for metric in self.metrics:\n            metric.to(self.device)\n            metric_value = metric(y_hat, y)\n            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)\n        return result\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.loss_fn(y_hat, y)\n        result = pl.EvalResult(checkpoint_on=loss)\n        result.log('val_loss', loss, on_epoch=True, on_step=False)\n        for metric in self.metrics:\n            metric.to(self.device)\n            metric_value = metric(y_hat, y)\n            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)\n        return result\n    \n    def test_step(self, batch, batch_idx):\n        y_hat = self(batch)\n        result = pl.EvalResult()\n        result.predictions = y_hat\n        return result","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:16.566940Z","iopub.status.busy":"2020-09-27T19:32:16.566078Z","iopub.status.idle":"2020-09-27T19:32:16.639391Z","shell.execute_reply":"2020-09-27T19:32:16.638899Z"},"papermill":{"duration":0.129994,"end_time":"2020-09-27T19:32:16.639493","exception":false,"start_time":"2020-09-27T19:32:16.509499","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class LungDataModule(pl.LightningDataModule):\n    def __init__(self, train_dataset, valid_dataset, test_dataset, batch_size=128):\n        super().__init__()\n        self.train_dataset = train_dataset\n        self.valid_dataset = valid_dataset\n        self.test_dataset = test_dataset\n        self.batch_size = batch_size\n        \n    def train_dataloader(self):\n        return DataLoader(train_dataset, shuffle=True, batch_size=self.batch_size)\n    \n    def val_dataloader(self):\n        return DataLoader(valid_dataset, shuffle=False, batch_size=self.batch_size)\n    \n    def test_dataloader(self):\n        return DataLoader(test_dataset, shuffle=False, batch_size=self.batch_size)\n    \n    \n\ndef predict_dataloader(model, dataloader):\n    predictions = []\n    model.eval()\n    for batch in dataloader:\n        predictions.append(model(batch).detach().cpu().numpy())\n    return np.vstack(predictions)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:16.738274Z","iopub.status.busy":"2020-09-27T19:32:16.737315Z","iopub.status.idle":"2020-09-27T19:32:16.806292Z","shell.execute_reply":"2020-09-27T19:32:16.806809Z"},"papermill":{"duration":0.123504,"end_time":"2020-09-27T19:32:16.806925","exception":false,"start_time":"2020-09-27T19:32:16.683421","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def quantile_loss(preds, target, quantiles, device):\n    #assert not target.requires_grad\n    #assert preds.size(0) == target.size(0)\n    losses = []\n    q = torch.tensor(quantiles).to(device).repeat(preds.shape[0], 1)\n    errors = target.unsqueeze(1).repeat(1, 3) - preds\n    losses = torch.max((q - 1) * errors, q * errors)\n    return torch.mean(losses).unsqueeze(0)\n\n\ndef metric_score(preds, target, max_fvc, min_fvc, device):\n    preds = (preds * (max_fvc - min_fvc)) + min_fvc\n    target = (target * (max_fvc - min_fvc)) + min_fvc\n    sigma = preds[:, 2] - preds[:, 0]\n    fvc_pred = preds[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = torch.max(sigma, torch.tensor(70.).to(device))\n    delta = torch.abs(target - fvc_pred)\n    delta = torch.min(delta, torch.tensor(1000.).to(device))\n    sq2 = torch.sqrt(torch.tensor(2.).to(device))\n    metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)\n    # print(torch.mean(metric))\n    return torch.mean(metric)\n\ndef metric_loss(preds, target, _lambda, quantiles, max_fvc, min_fvc, device):\n    return ((_lambda * quantile_loss(preds, target, quantiles, device)) + \n            ((1 - _lambda) * metric_score(preds, target, max_fvc, min_fvc, device)))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CV Training"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-27T19:32:16.905069Z","iopub.status.busy":"2020-09-27T19:32:16.904231Z","iopub.status.idle":"2020-09-27T19:32:28.416016Z","shell.execute_reply":"2020-09-27T19:32:28.415491Z"},"papermill":{"duration":11.564898,"end_time":"2020-09-27T19:32:28.416165","exception":false,"start_time":"2020-09-27T19:32:16.851267","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"seed_everything(2020)\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n_skf = StratifiedKFold(n_splits=5)\n_kf = KFold()\nlr_monitor_callback = LearningRateLogger(logging_interval='step')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(2020)\n\nname = 'tabnet_img_featuresall_skf_mloss1_lr5e-3_faster_missing'\ntrained_models = []\ncv_predictions = []\nepochs = 350\nbatch_size = 128\n\n\nfor i, (train_dataset, valid_dataset) in enumerate(get_splits(patient_df, train, image_feature_df, _skf, 'expected_fvc',expand=True)):\n    total_steps = epochs * math.ceil((len(train_dataset) / batch_size))\n    train_params = train_dataset.get_params()\n    loss_fn = partial(\n        metric_loss,\n        _lambda=1,\n        quantiles=(0.3, 0.5, 0.7),\n        max_fvc=train_params['max_fvc'],\n        min_fvc=train_params['min_fvc'],\n        device=device\n    )\n    pathdir = Path(f'./models/{name}/fold_{i}/')\n    pathdir.mkdir(parents=True, exist_ok=True)\n    checkpoint_callback = ModelCheckpoint(\n        filepath=pathdir,\n        save_top_k=1,\n        save_last=True,\n        verbose=False,\n        monitor='val_nll',\n        mode='min',\n        prefix=''\n    )\n\n    cv_result_tracker = CVResultTracker('./fold_results.csv', './model_results.csv')\n    model = LTabnet(5e-3, total_steps, train_params, in_tabular_features=29).to(device)\n    model.set_loss_fn(loss_fn)\n    test_dataset = LungDataset('test', test, None, train_params)\n    test_dataset.set_inference(True)\n    data_module = LungDataModule(train_dataset, valid_dataset, test_dataset, batch_size=batch_size)\n    \n    tb_logger = loggers.TensorBoardLogger('lightning_logs/', name=f'{name}', version=f'fold_{i}')\n    csv_logger = loggers.CSVLogger('results', name=name, version=f'fold_{i}')\n    \n    trainer = pl.Trainer(\n        max_epochs=epochs, \n        check_val_every_n_epoch=5, \n        logger=[tb_logger, csv_logger],\n        checkpoint_callback=checkpoint_callback,\n        gpus=1,\n        callbacks=[lr_monitor_callback, cv_result_tracker]\n    )\n    trainer.fit(model, data_module)\n    trained_models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.050607,"end_time":"2020-09-27T19:32:30.671580","exception":false,"start_time":"2020-09-27T19:32:30.620973","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!ls ./models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ./results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}