{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.metrics import mean_absolute_error\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Regular Imports\nimport os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom tabulate import tabulate\nimport missingno as msno \nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\nfrom scipy.stats import pearsonr\n\nimport pydicom # for DICOM images\nfrom skimage.transform import resize\nimport copy\nimport re\n\n# Segmentation\nfrom glob import glob\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport scipy.ndimage\nfrom skimage import morphology\nfrom skimage import measure\nfrom skimage.transform import resize\nfrom sklearn.cluster import KMeans\nfrom plotly import __version__\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom plotly.tools import FigureFactory as FF\nfrom plotly.graph_objs import *\ninit_notebook_mode(connected=True) \n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read train csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\ndf = pd.read_csv(base_path + 'train.csv')\n# df.sample(5,  random_state=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Processing image data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create base director for Train .dcm files\ndirector = \"../input/osic-pulmonary-fibrosis-progression/train\"\n\n# Create path column with the path to each patient's CT\ndf[\"Path\"] = director + \"/\" + df[\"Patient\"]\n\n# Create variable that shows how many CT scans each patient has\ndf[\"CT_number\"] = 0\n\nfor k, path in enumerate(df[\"Path\"]):\n    df[\"CT_number\"][k] = len(os.listdir(path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df[df['Patient']!='ID00011637202177653955184']\ndf = df[df['Patient']!='ID00052637202186188008618']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mid_ct_scan(patient_dir):\n    # First Order the files in the dataset\n    files = []\n    for dcm in list(os.listdir(patient_dir)):\n        files.append(dcm) \n    files.sort(key=lambda f: int(re.sub('\\D', '', f)))\n\n    # Read the middle image in the Dataset\n    mid_ct_scan = len(files)//2\n    dcm = files[mid_ct_scan]\n    path = patient_dir + \"/\" + dcm\n    datasets = pydicom.dcmread(path)\n    img = datasets.pixel_array/2000 #normalize\n    img = cv2.resize(img, (224,224))\n    #     plt.imshow(img, cmap='plasma')\n#     img = img.flatten()\n\n    return img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['mid_ct_scan'] = df['Path'].apply(lambda x: get_mid_ct_scan(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check img of random patient\nimg = df['mid_ct_scan'][78]\nplt.imshow(img, cmap='plasma')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering for csv data"},{"metadata":{},"cell_type":"markdown","source":"1. Get number of weeks passed from first measurement"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_weeks_passed(df):\n    min_week_dict = df.groupby('Patient').min('Weeks')['Weeks'].to_dict()\n    df['MinWeek'] =  df['Patient'].map(min_week_dict)\n    df['WeeksPassed'] = df['Weeks'] - df['MinWeek']\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"2. Get first FVC measurement"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_baseline_FVC(df):\n    _df = (\n        df\n        .loc[df.Weeks == df.MinWeek][['Patient','FVC']]\n        .rename({'FVC': 'FirstFVC'}, axis=1)\n        .groupby('Patient')\n        .first()\n#         .reset_index()\n    )\n    \n    first_FVC_dict = _df.to_dict()['FirstFVC']\n    df['FirstFVC'] =  df['Patient'].map(first_FVC_dict)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3. Get Height"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_height(row):\n    if row['Sex'] == 'Male':\n        return row['FirstFVC'] / (27.63 - 0.112 * row['Age'])\n    else:\n        return row['FirstFVC'] / (21.78 - 0.101 * row['Age'])\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = get_weeks_passed(df)\ndf = get_baseline_FVC(df)\ndf['Height'] = df.apply(calculate_height, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set up data processing pipeline for csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the necessary Encoders & Transformers\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\nfrom sklearn.compose import ColumnTransformer\n\n# define which attributes shall not be transformed, are numeric or categorical\nno_transform_attribs = ['Patient', 'Weeks', 'MinWeek','mid_ct_scan','FVC']\nnum_attribs = ['Percent', 'Age', 'WeeksPassed', 'FirstFVC','Height']\ncat_attribs = ['Sex', 'SmokingStatus']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\n\nclass NoTransformer(BaseEstimator, TransformerMixin):\n    \"\"\"Passes through data without any change and is compatible with ColumnTransformer class\"\"\"\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        assert isinstance(X, pd.DataFrame)\n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## transform features into series\n\n# create an instance of the ColumnTransformer\ndatawrangler = ColumnTransformer(([\n     # the No-Transformer does not change the data and is applied to all no_transform_attribs \n     ('original', NoTransformer(), no_transform_attribs),\n     # Apply StdScaler to the numerical attributes, here you can change to e.g. MinMaxScaler()   \n     ('StdScaler', StandardScaler(), num_attribs),\n     # OneHotEncoder all categorical attributes.   \n     ('cat_encoder', OneHotEncoder(), cat_attribs),\n    ]))\n\ntransformed_data_series = []\ntransformed_data_series = datawrangler.fit_transform(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## put transformed series into dataframe\n\n# get column names for non-categorical data\nnew_col_names = no_transform_attribs + num_attribs\n\n# extract possible values from the fitted transformer\ncategorical_values = [s for s in datawrangler.named_transformers_[\"cat_encoder\"].get_feature_names()]\nnew_col_names += categorical_values\n\n# create Dataframe based on the extracted Column-Names\ntrain_sklearn_df = pd.DataFrame(transformed_data_series, columns=new_col_names)\ntrain_sklearn_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train test split"},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_features_list = ['Percent','Age','WeeksPassed','FirstFVC','Height','x0_Female','x1_Currently smokes','x1_Ex-smoker']\nctscan_features_list = ['mid_ct_scan']\n\nX = train_sklearn_df[csv_features_list].astype(float)\nX['mid_ct_scan'] = train_sklearn_df[ctscan_features_list]\n\ny = train_sklearn_df[['FVC']].astype(float)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n\nX_train_img = X_train[ctscan_features_list]\n# X_train_img = X_train_img['mid_ct_scan']\nX_train_csv = X_train[csv_features_list]\n\nX_test_img = X_test[ctscan_features_list]\n# X_test_img = X_test_img['mid_ct_scan']\nX_test_csv = X_test[csv_features_list]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_img = X_train_img['mid_ct_scan'].to_numpy()\nX_train_img = np.stack( X_train_img, axis=0 )\n\n\nX_test_img = X_test_img['mid_ct_scan'].to_numpy()\nX_test_img = np.stack( X_test_img, axis=0 )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Y-shaped NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.layers import Flatten\nfrom tensorflow.keras.layers import Conv2D\nfrom tensorflow.keras.layers import MaxPooling2D\nfrom tensorflow.keras.layers import concatenate\nfrom tensorflow.keras.utils import plot_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to-do pre-trained ResNet / VGG","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def createDualInputModel():\n    \n    # Left for image\n    Lin = Input(shape=(224,224,1), name = 'ctscan')\n    Lx = Conv2D(32,(3,3),padding='same',activation='relu')(Lin)\n    Lx = MaxPooling2D(pool_size=(2,2))(Lx)\n    Lx = Flatten()(Lx)\n    Lx = Dense(128,activation='relu')(Lx)\n    \n    # Right for csv\n    Rin = Input((len(csv_features_list),), name = \"csv\")\n    Rx = Dense(128,activation='relu')(Rin)\n    \n    # concatenate\n    x = concatenate([Lx,Rx],axis=-1)\n    x = Dense(128,activation='relu')(x)\n    x = Dense(64,activation='relu')(x)\n    x = Dense(1, activation='linear')(x) # no activation function since regression problem\n    \n    model = Model(inputs=[Lin,Rin],outputs=x)\n    model.compile(loss='mean_absolute_error',\n                  optimizer='rmsprop',\n                  metrics=['mean_absolute_error'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nn_model = createDualInputModel()\nnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nn_model.fit([X_train_img,X_train_csv], y_train,\n             validation_data=([X_test_img,X_test_csv], y_test),\n             epochs=10,\n             batch_size=32,\n             shuffle=True) #, callbacks=callbacks_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train XGBoost model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport tensorflow as tf\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed): \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\nseed_everything(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_list = ['Percent','Age','WeeksPassed','FirstFVC','Height','x0_Female','x1_Currently smokes','x1_Ex-smoker']\n\nX = train_sklearn_df[features_list].astype(float)\ny = train_sklearn_df[['FVC']].astype(float)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xg_reg = xgb.XGBRegressor(objective ='reg:squarederror',\n                          colsample_bytree = 0.3,\n                          learning_rate = 0.1,\n                          max_depth = 5,\n                          alpha = 10,\n                          n_estimators = 10)\nxg_reg.fit(X_train, y_train)\npreds = xg_reg.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mae = mean_absolute_error(y_test, preds)\nmae","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare submission csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\ndf = pd.read_csv(base_path + 'test.csv')\ndf = df.rename(columns={'Weeks':'MinWeek'})\n\nbase_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\ndf_submission = pd.read_csv(base_path + 'sample_submission.csv')\ndf_submission\n\n# merge predictions from test set into submission set\ndf_submission[['Patient','Weeks']] = df_submission['Patient_Week'].str.split(\"_\",expand=True,)\ndf_submission['Weeks'] = df_submission['Weeks'].astype('int')\n# df = df.drop(['Weeks'], axis=1)\ndf_submission = df_submission.drop(['FVC','Confidence'],axis=1)\ndf_submission = pd.merge(df_submission, df, on=['Patient'], how='left')\ndf_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create base director for test .dcm files\ndirector = \"../input/osic-pulmonary-fibrosis-progression/test\"\n\n# Create path column with the path to each patient's CT\ndf_submission[\"Path\"] = director + \"/\" + df_submission[\"Patient\"]\n\n# Create variable that shows how many CT scans each patient has\ndf_submission[\"CT_number\"] = 0\n\nfor k, path in enumerate(df_submission[\"Path\"]):\n    df_submission[\"CT_number\"][k] = len(os.listdir(path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['mid_ct_scan'] = df_submission['Path'].apply(lambda x: get_mid_ct_scan(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feature Engineering\ndf_submission['WeeksPassed'] = df_submission['Weeks'] - df_submission['MinWeek']\ndf_submission = get_baseline_FVC(df_submission)\ndf_submission['Height'] = df_submission.apply(calculate_height, axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission1 = df_submission[['Patient','Weeks','FVC','Percent','Age','Sex','SmokingStatus','Path','CT_number','mid_ct_scan','MinWeek','WeeksPassed','FirstFVC','Height']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transformed_data_series = datawrangler.transform(df_submission1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission1['Patient_Week'] = df_submission['Patient_Week']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submssions = df_submission1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## put transformed series into dataframe\n\n# get column names for non-categorical data\nnew_col_names = no_transform_attribs + num_attribs\n\n# extract possible values from the fitted transformer\ncategorical_values = [s for s in datawrangler.named_transformers_[\"cat_encoder\"].get_feature_names()]\nnew_col_names += categorical_values\n\n# create Dataframe based on the extracted Column-Names\ntrain_sklearn_df = pd.DataFrame(transformed_data_series, columns=new_col_names)\ntrain_sklearn_df.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_features_list = ['Percent','Age','WeeksPassed','FirstFVC','Height','x0_Female','x1_Currently smokes','x1_Ex-smoker']\nctscan_features_list = ['mid_ct_scan']\n\nX = train_sklearn_df[csv_features_list].astype(float)\nX['mid_ct_scan'] = train_sklearn_df[ctscan_features_list]\n\n# y = train_sklearn_df[['FVC']].astype(float)\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n\nX_train_img = X[ctscan_features_list]\n# X_train_img = X_train_img['mid_ct_scan']\nX_train_csv = X[csv_features_list]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_img = X_train_img['mid_ct_scan'].to_numpy()\nX_train_img = np.stack( X_train_img, axis=0 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = nn_model.predict([X_train_img,X_train_csv],\n             batch_size=32) #, callbacks=callbacks_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['FVC'] = preds\ndf_submission['Confidence'] = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = df_submission[['Patient_Week','FVC','Confidence']]\ndf_submission.to_csv('/kaggle/working/submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def create_cnn_model():\n#     Lin = Input(shape=(224,224,1), name = 'ctscan')\n#     Lx = Conv2D(1, (3,3),padding='same',activation='relu')(Lin)\n#     Lx = MaxPooling2D(pool_size=(2,2))(Lx)\n#     Lx = Flatten()(Lx)\n#     Lx = Dense(64)(Lx)\n#     Lx = Dense(1)(Lx)\n\n    \n#     model = Model(inputs=Lin,outputs=Lx)\n#     model.compile(loss='categorical_crossentropy',\n#                   optimizer='rmsprop',\n#                   metrics=['accuracy'])\n    \n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cnn_model = create_cnn_model()\n# cnn_model.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cnn_model.fit(X_train_img,y_train,\n#               validation_data=(X_test_img, y_test),\n#               epochs=10,\n#               batch_size=16,\n#               shuffle=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}