{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import os.path\nimport random\nimport numpy as np\nfrom skimage import io\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nimport math  \nimport pandas as pd\nimport sklearn.metrics as sm\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Check current directory\ncurr_wkdir = os.getcwd()          # get the current working directory\nprint(curr_wkdir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Load the model\nimport os.path\nfrom tensorflow.keras.models  import  load_model\n\n# model_path = curr_wkdir + '/model'           # *** parameter : need to update with correct folder name during submission\n# model_path = model_path.replace(\"\\\\\",\"/\")\nmodel_path = '../input/osic-model-ver3c'\nmodel_savefile = model_path + '/osic_model_ver1.0.h5'\n\nosic_model = load_model(model_savefile)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"osic_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check model weights\nosic_model.get_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# **************************************************************\n# *** (B) Testing the model on data with known true values   ***\n# **************************************************************\n# Test with a known true values dataset, compute standard deviation, compute model's confidence, \n# and create submission.csv\n\n# p = curr_wkdir + '/data'              # *** parameter : need to update with correct folder name during submission         \n# test_dir = p.replace(\"\\\\\",\"/\")\ntest_dir ='../input/osic-pulmonary-fibrosis-progression'\n\n# Changes on 5/Oct/2020  (train_data for reference)\nmaxPercent = 160\n# need to refer to train data for data attributes profiling     5/Oct/2020\ntrain_file = test_dir + '/train.csv'   # *** parameter : need to update with correct file name during submission\ntrain_data = pd.read_csv(train_file)\ntrain_data['FullFVC'] = (train_data['FVC'] * 100) / train_data['Percent'] \ntrain_data['PercentScaled'] = train_data['Percent'] / maxPercent\ntrain_data['Gender'] = np.where(train_data['Sex'] == 'Male', 1, 0)\n# end of changes for train_data profile  5/Oct/2020\n\n\n# *** Specify the file to be tested below  5/Oct/2020   \n# test_file = '../input/osic-pulmonary-fibrosis-progression/train.csv'   # *** parameter : need to update with correct file name during submission\ntest_file = '../input/osic-pulmonary-fibrosis-progression/test.csv'   # *** parameter : need to update with correct file name during submission\n\n\ntest_data = pd.read_csv(test_file)\n\n# below codes is to remove those rows belonging to the compressed dicom images ID00011637202177653955184\nif  'ID00011637202177653955184' in test_data.values:\n    # Get names of indexes for which column Patient has specified value \n    indexNames = test_data[ test_data['Patient'] == \"ID00011637202177653955184\" ].index\n    # Delete these row indexes from dataFrame\n    test_data.drop(indexNames , inplace=True)\n\n# Add computed columns to the dataframe - FullFVC, PercentScaled, Gender\ntest_data['FullFVC'] = (test_data['FVC'] * 100) / test_data['Percent'] \n\n# Changes for ver3c\n# test_data['PercentScaled'] = test_data['Percent'] / 100\n# maxPercent = 160                                                   # 4/Oct/2020  - move to the top\ntest_data['PercentScaled'] = test_data['Percent'] / maxPercent\n# end of changes for ver3c\n\ntest_data['Gender'] = np.where(test_data['Sex'] == 'Male', 1, 0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Below statements are for testing and checking\nprint(test_data.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate the test patient info and test input file\ntest_patient_info = test_data.drop(columns=['Weeks','FVC','Percent','Sex','PercentScaled'])\nprint(test_patient_info.columns.values)\nprint(len(test_patient_info))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate the test patient info and test input file\ntest_patient_info_np = np.array(test_patient_info)\npatient_count = len(test_patient_info)\n\ntest_data_file = pd.DataFrame()\n\nfor i in range(patient_count):\n    patient = test_patient_info_np[i,0]\n    age = test_patient_info_np[i,1]\n    smokingstatus = test_patient_info_np[i,2]\n    fullfvc = test_patient_info_np[i,3]\n    gender = test_patient_info_np[i,4]\n\n    for wk in range(-12, 134):                     # need to generate from weeks -12 to 133\n        test_data_file = (test_data_file.append({'Patient':patient, 'Weeks':wk,\n                                                'Age':age, 'SmokingStatus':smokingstatus,\n                                                'FullFVC':fullfvc, 'Gender':gender},\n                                                ignore_index=True))\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The below codes are for testing\nprint(f'test_data_file.columns.values : {test_data_file.columns.values}')\nprint(f'len(test_data_file): {len(test_data_file)}')\n# print(test_data_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# function for plotting images\ndef plotImages(images_arr):\n    fig, axes = plt.subplots(1, 10, figsize=(20,20))\n    axes = axes.flatten()\n    for img, ax in zip( images_arr, axes):\n        ax.imshow(img)\n        ax.axis('off')\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Backup - NOTE : this is for ver1 & Ver3\n# Define function that returns montage of images\ndef create_montage(dir):\n    inputImages = []\n    outputImage = np.zeros((224,224), dtype=\"uint8\")      \n    files_list = []\n    \n    # obtain the list of image files in the folder\n    files_list = os.listdir(dir)\n\n    # Check number of files (i.e. images in the folder)\n    onlyfiles = next(os.walk(dir))[2] #dir is your directory path as string\n    imagecount = len(onlyfiles)\n\n    # based on the number of images for the patient, determine the interval if 4 images are to be taken\n    imageinterval = math.floor(imagecount/4) \n\n    # initialize array for selected images\n    selectedimagesno= []           \n\n    # Loop thru 0 to 3, and take images file no for the interval\n    for i in range(4):\n        imageslice = (i * imageinterval) + 1\n        selectedimagesno.append(imageslice) \n\n    # print (str(selectedimagesno)) \n\n    # loop thru the selected image file numbers and append image to inputImages array\n    for item in selectedimagesno:\n#         print(f'{dir}/{files_list[item]}')         # for testing purpose\n        image = io.imread(f'{dir}/{files_list[item]}')\n#         print(f'{dir}/{item}.dcm')                 # for testing purpose\n        # plt.imshow(image)\n        image = cv2.resize(image,(112,112))\n        inputImages.append(image)\n\n\n    # # inputImages (below plot image statement works!)\n    # plotImages(inputImages)\n\n    outputImage[0:112, 0:112] = inputImages[0]\n    outputImage[0:112, 112:224] = inputImages[1]\n    outputImage[112:224, 112:224] = inputImages[2]\n    outputImage[112:224, 0:112] = inputImages[3]  \n\n\n# # Below plot images statement works!\n#     plt.imshow(outputImage)\n    \n    return outputImage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Define function that returns montage of images (for ver2)\n# def create_montage(dir):\n#     inputImages = []\n#     outputImage = np.zeros((312,312), dtype=\"uint8\")      \n#     files_list = []\n    \n#     # obtain the list of image files in the folder\n#     files_list = os.listdir(dir)\n\n#     # Check number of files (i.e. images in the folder)\n#     onlyfiles = next(os.walk(dir))[2] #dir is your directory path as string\n#     imagecount = len(onlyfiles)\n\n#     # based on the number of images for the patient, determine the interval if 4 images are to be taken\n#     imageinterval = math.floor(imagecount/4) \n\n#     # initialize array for selected images\n#     selectedimagesno= []           \n\n#     # Loop thru 0 to 3, and take images file no for the interval\n#     for i in range(4):\n#         imageslice = (i * imageinterval) + 1\n#         selectedimagesno.append(imageslice) \n\n#     # print (str(selectedimagesno)) \n\n#     # loop thru the selected image file numbers and append image to inputImages array\n#     for item in selectedimagesno:\n# #         print(f'{dir}/{files_list[item]}')         # for testing purpose\n#         image = io.imread(f'{dir}/{files_list[item]}')\n# #         print(f'{dir}/{item}.dcm')                 # for testing purpose\n#         # plt.imshow(image)\n#         image = cv2.resize(image,(156,156))\n#         inputImages.append(image)\n\n\n#     # # inputImages (below plot image statement works!)\n#     # plotImages(inputImages)\n\n#     outputImage[0:156, 0:156] = inputImages[0]\n#     outputImage[0:156, 156:312] = inputImages[1]\n#     outputImage[156:312, 156:312] = inputImages[2]\n#     outputImage[156:312, 0:156] = inputImages[3]  \n\n\n# # # Below plot images statement works!\n# #     plt.imshow(outputImage)\n    \n#     return outputImage\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Loop thru the image folder and create montage for every folder\n\n# p = curr_wkdir + '/data/train'        # *** parameter : need to update with correct folder name during submission\n# path = p.replace(\"\\\\\",\"/\")\n\n\n# *** Specify the folder for images to be tested below   5/Oct/2020\n# path = '../input/osic-pulmonary-fibrosis-progression/train'\npath = '../input/osic-pulmonary-fibrosis-progression/test'\nprint(path)\n\nimages_list=[]\nimagesID_list =[]\ndirectory_list = []\n\ndirectory_contents = os.listdir(path)\n# print(directory_contents)       # for testing\n\n# the case ID00011637202177653955184 with compressed dicom images will be removed \n# because the quality of the decompressed image may be compromised, hence affecting \n# the training, validation and testing\n# remove the folder with compressed dicom from dataset\ndirectory_list = directory_contents\n\nif directory_list.count('ID00011637202177653955184') > 0:\n    directory_list.remove('ID00011637202177653955184')\n\nfor item in directory_contents:\n#     print(item)\n#     print(f'{path}/{item}')\n    folder = f'{path}/{item}'\n#     print(folder)\n#     montage = create_montage(f'{path}/{item}')\n    montage = create_montage(folder)\n    images_list.append(montage)\n    imagesID_list.append(item)\n\n\n# plt.imshow(images_list)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"plotImages(images_list)            # for testing purpose","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Backup before changes 5/Oct/2020 - point to generated test data file\n# # prepare input images\n# # Prepare image list for the test set\n\n# input_images = []\n\n# for i in range(len(test_data)) : \n#     imageloc = imagesID_list.index(test_data.iloc[i, 0])      \n#     input_images.append(images_list[imageloc])\n\n# input_images = np.array(input_images)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare input images\n# Prepare image list for the test set\n\ninput_images = []\n\nfor i in range(len(test_data_file)) : \n    imageloc = imagesID_list.index(test_data_file.iloc[i, 3])     #patient column in test_data_file  \n    input_images.append(images_list[imageloc])\n\ninput_images = np.array(input_images)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Process the attributes of the patients\n# import the necessary packages\n\n# a) Numeric/continuous values:    Age, Weeks, FullFVC\n# b) Categorical values:           Gender and Smoking Status \n\n# data in datasets are as following:\n# ['Weeks' 'Age' 'SmokingStatus' 'FullFVC' 'Gender']\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.preprocessing import MinMaxScaler\n\ndef process_input_attributes(df, inputdata):\n    # initialize the column names of the continuous data\n    continuous = ['Weeks','Age','FullFVC']\n    \n    # performin min-max scaling each continuous feature column to\n    # the range [0, 1]\n    cs = MinMaxScaler()\n    inputdataContinuous = cs.fit_transform(inputdata[continuous])\n    \n    # one-hot encode the SmokingStatus and Gender categorical data (by definition of\n    # one-hot encoding, all output features are now in the range [0, 1])\n    zipBinarizer = LabelBinarizer().fit(df['SmokingStatus'])\n    inputdataCategorical1 = zipBinarizer.transform(inputdata['SmokingStatus'])\n\n    zipBinarizer = LabelBinarizer().fit(df['Gender'])\n    inputdataCategorical2 = zipBinarizer.transform(inputdata['Gender'])\n    \n    # construct our inputdataing and testing data points by concatenating\n    # the categorical features with the continuous features\n    inputdataX = np.hstack([inputdataCategorical1, inputdataCategorical2, inputdataContinuous])\n    \n    # return the concatenated inputdataing and testing data\n    return (inputdataX)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Backup before changes 5/Oct/2020 - point to generated test data file\n# # prepare input data\n# # ['Weeks' 'Age' 'SmokingStatus' 'FullFVC' 'Gender']\n\n# input_data = test_data.drop(columns=['Patient','FVC','Percent','Sex','PercentScaled'])\n\n# # *** Below changes was made on 4/Oct/2020\n# # input_attributes = process_input_attributes(test_data, input_data)\n# input_attributes = process_input_attributes(train_data, input_data)\n# # *** end of change  4/Oct/2020\n\n# print(f'input data cols : {input_data.columns.values}')   # for checking purpose","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare input data     -   point to test_data_file  5/Oct/2020\n# test_data_file.columns.values : ['Age' 'FullFVC' 'Gender' 'Patient' 'SmokingStatus' 'Weeks']\n# ['Weeks' 'Age' 'SmokingStatus' 'FullFVC' 'Gender']\n\ninput_data = test_data_file.drop(columns=['Patient'])\ninput_attributes = process_input_attributes(train_data, input_data)       # chg on 4/Oct/2020\n\nprint(f'input data cols : {input_data.columns.values}')   # for checking purpose","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ***  TESTING : The below paragrapgh are for testing\nprint(f'len(test_data) : {len(test_data)}')\nprint(f'len(test_data_file) : {len(test_data_file)}')\nprint(f'len(input_images) : {len(input_images)}')\nprint(f'len(input_attributes) : {len(input_attributes)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# make prediction with model\ntest_predictions = osic_model.predict([input_attributes, input_images])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # The below para is not required anymore because the true value is not known   5/Oct/2020\n# # Measure accuracy of prediction\n# actual_percentscaled = test_data['PercentScaled']\n\n# # print(f'{actual_percentscaled}')     # for testing\n# # print(f\"Explain variance score = {round(sm.explained_variance_score(y_valid_rs, predict_validations), 2)}\") \n\n# print(f\"Explain variance score = {round(sm.explained_variance_score(actual_percentscaled, test_predictions), 2)}\") \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Backup before making changes to point to test_data_file \n# #      (generated based on weeks -12 to 133)  5/Oct/2020\n# # Try 3a   ==> works but column in alphabetical order\n# # # include predictions into the test data, compute predicted FVC, standard deviation, \n# # # and model's confidence for every record\n# # # Generate output submission.csv file\n# # # Data used are from test_data & test_predictions\n\n# # column_names = ['Patient' 'Weeks' 'FVC' 'Percent' 'Age' 'Sex' 'SmokingStatus' 'FullFVC'\n# #  'PercentScaled' 'Gender','PredictionPercentScaled']\n\n# test_data_np = np.array(test_data)\n# # results = pd.DataFrame(columns = column_names)\n# results = pd.DataFrame()\n# rec_count = len(test_data)\n\n# for i in range(rec_count):\n#     results = (results.append({'Patient':test_data_np[i,0], 'Weeks':test_data_np[i,1], \n#                                'FVC': test_data_np[i,2], 'Percent': test_data_np[i,3], \n#                                'Age': test_data_np[i,4], 'Sex': test_data_np[i,5], \n#                                'SmokingStatus': test_data_np[i,6], 'FullFVC': test_data_np[i,7], \n#                                'PercentScaled': test_data_np[i,8], 'Gender': test_data_np[i,9], \n#                                'PredictedPercentScaled': test_predictions[i]}, \n#                               ignore_index=True))    \n    \n# print('done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Point to test_data_file  5/Oct/2020\n# Try 3a   ==> works but column in alphabetical order\n# # include predictions into the test data, compute predicted FVC, standard deviation, \n# # and model's confidence for every record\n# # Generate output submission.csv file\n# # Data used are from test_data & test_predictions\n\n# test_data_file.columns.values : ['Age' 'FullFVC' 'Gender' 'Patient' 'SmokingStatus' 'Weeks']\n#  'PercentScaled' 'Gender','PredictionPercentScaled']\n\ntest_data_file_np = np.array(test_data_file)\n# results = pd.DataFrame(columns = column_names)\nresults = pd.DataFrame()\nrec_count = len(test_data_file)\n\nfor i in range(rec_count):\n    results = (results.append({'Age':test_data_file_np[i,0], 'FullFVC':test_data_file_np[i,1], \n                               'Gender': test_data_file_np[i,2], 'Patient': test_data_file_np[i,3], \n                               'SmokingStatus': test_data_file_np[i,4], 'Weeks': test_data_file_np[i,5], \n                               'PredictedPercentScaled': test_predictions[i]}, \n                              ignore_index=True))    \n    \nprint('done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print(len(test_data_file))    # for testing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Backup before changes due to generated test data file\n# #   ==> true fvc will not be known, hence unable to compute std deviation and confidence\n\n# # compute predicted full fvc\n# # compute mean square error and std dev\n\n# # Changes for ver3c\n# # results['PredictedFVC'] = results['PredictedPercentScaled'] * results['FullFVC']\n# results['PredictedFVC'] = ((results['PredictedPercentScaled'] * maxPercent) / 100) * results['FullFVC']\n# # end of changes for ver3c\n\n# predictedfvc = results['PredictedFVC']\n# truefvc = results['FVC']\n\n# # Compute Mean square error\n# print(f\"Mean squared error = {round(sm.mean_squared_error(truefvc, predictedfvc), 2)}\") \n# print(f\"Standard deviation = {round(math.sqrt(sm.mean_squared_error(truefvc, predictedfvc)), 5)}\") \n\n# print(f\"Explain variance score = {round(sm.explained_variance_score(truefvc, predictedfvc), 5)}\") \n\n# std_deviation = math.sqrt(sm.mean_squared_error(truefvc, predictedfvc))\n# # print(f'std_deviation : {std_deviation}')                 \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Point to test_data_file (generated test data file)    5/Oct/2020\n# compute predicted full fvc\n\n# Changes for ver3c\n# results['PredictedFVC'] = results['PredictedPercentScaled'] * results['FullFVC']\nresults['PredictedFVC'] = ((results['PredictedPercentScaled'] * maxPercent) / 100) * results['FullFVC']\n# end of changes for ver3c\n\npredictedfvc = results['PredictedFVC']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Compute confidence not required any more because true fvc is not known   5/Oct/2020\n# # Compute Confidence of prediction for every row\n# def compute_confidence(std_dev, true_fvc, predicted_fvc):\n#     std_dev_clipped = max(std_dev, 70)\n    \n# #     *** Changes made 5/OCt/2020\n#     upper_limit = np.array([1000])\n# #     delta =  min(abs(true_fvc - predicted_fvc),1000)\n#     delta =  min(abs(true_fvc - predicted_fvc),upper_limit) \n# #     *** end of changes made 5/Oct/2020\n    \n#     sqrt_2 = math.sqrt(2)\n    \n#     metric = -((sqrt_2*delta)/std_dev_clipped) - math.log(sqrt_2*std_dev_clipped)\n#     return metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# # Backup before making changes due to generated test data file   5/Oct/2020\n# # # Compute Confidence in results sets\n# # results['Confidence'] = compute_confidence(std_deviation,results['FVC'],results['PredictedFVC'])\n\n# results_confidence =[]\n# results_np = np.array(results)\n\n# for i in range(len(results_np)):\n#     truefvc       = results_np[i,1]     # i is the row number\n#     predictedfvc  = results_np[i,11]  \n#     confidence = compute_confidence(std_deviation, truefvc, predictedfvc)\n#     results_confidence.append(confidence)\n    \n# # print(f'results_confidence : {results_confidence}')       # for testing\n\n# # add the results_confidence as the 'Confidence' column in results\n# results['Confidence'] = results_confidence\n\n# # for checking / testing\n# # print(results_confidence)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Compute Confidence in results sets\n# ==> Confidence will be hardcoded to 100 because true fvc is not known\n\nresults_confidence =[]\nresults_np = np.array(results)\n\nfor i in range(len(results_np)):\n    confidence = 100\n    results_confidence.append(confidence)\n    \n# print(f'results_confidence : {results_confidence}')       # for testing\n\n# add the results_confidence as the 'Confidence' column in results\nresults['Confidence'] = results_confidence\n\n# for checking / testing\n# print(results_confidence)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print(f'results columns : {results.columns.values}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Generate the column Patient_Week \n# convert week to int first to remove .0 before converting to string\nresults['Patient_Week'] = results['Patient'] + '_' + results['Weeks'].astype(int).astype(str)\nprint(f'results columns : {results.columns.values}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Below statement is for testing - for review of the results\n# results_filename = curr_wkdir + '/results.csv'\n# results_filename = results_filename.replace(\"\\\\\",\"/\")\nresults_filename = './' + 'results.csv'\nresults.to_csv(results_filename, index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Below statements are for testing\n# print(results['Weeks'])\n# print(results['Patient'])\n# print(results['Patient_Week'])\nprint(f'results no of rows : {len(results)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Generate submission file\nsubmission = pd.DataFrame()\npatientwk_s = results['Patient_Week']\nfvc_s       = results['PredictedFVC']\nconfidence_s = results['Confidence']\n\nsubmission = pd.concat([patientwk_s, fvc_s, confidence_s], axis=1) \n\n# rename submssion file column\nsubmission_file = submission.rename(columns = {'PredictedFVC': 'FVC'}, inplace = False)\n\n# remove square bracket from the fvc and confidence columns\n# submission_file['FVC'] =  submission_file['FVC'].str.get(0)                          \nsubmission_file['FVC'] =  (submission_file['FVC'].str.get(0)).round(0).astype(int)     # 5/Oct/2020\n# submission_file['Confidence'] =  submission_file['Confidence'].str.get(0)            # chg due to generated data file 5/Oct/2020\n\nprint(submission_file)\n\n# output submission file to csv, set index = False so that index will not be included in output\n# output_filename = curr_wkdir + '/submission.csv'                   #parameter : need to update with correct folder name\n# output_filename = output_filename.replace(\"\\\\\",\"/\")\n\noutput_filename = './' + 'submission.csv'\n# output_filename = '../output' + 'submission.csv'\n\nsubmission_file.to_csv(output_filename, index=False) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}