{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pydicom\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler,LabelEncoder,OneHotEncoder,PowerTransformer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading the Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"main_dir = '../input/osic-pulmonary-fibrosis-progression'\n\ntrain_files = tf.io.gfile.glob(main_dir+\"/train/*/*\")\ntest_files = tf.io.gfile.glob(main_dir+\"/test/*/*\")\n\nsample_sub = pd.read_csv(main_dir+'/sample_submission.csv')\ntrain = pd.read_csv(main_dir + \"/train.csv\")\ntest = pd.read_csv(main_dir + \"/test.csv\")\n\nprint (\"Number of train patients: {}\\nNumber of test patients: {:4}\"\n       .format(train.Patient.nunique(), test.Patient.nunique()))\n\nprint (\"\\nTotal number of Train patient records: {}\\nTotal number of Test patient records: {:6}\"\n       .format(len(train_files), len(test_files)))\n\ntrain.shape, test.shape, sample_sub.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Function to calculate Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"def laplace_log_likelihood(y_true, y_pred, sigma=70):\n    # values smaller than 70 are clipped\n    sigma_clipped = tf.maximum(sigma, 70)\n\n    # errors greater than 1000 are clipped\n    delta_clipped = tf.minimum(tf.abs(y_true - y_pred), 1000)\n    \n    # type cast them suitably\n    delta_clipped = tf.cast(delta_clipped, dtype=tf.float32)\n    sigma_clipped = tf.cast(sigma_clipped, dtype=tf.float32)\n    \n    # score function\n    score = - tf.sqrt(2.0) * delta_clipped / sigma_clipped - tf.math.log(tf.sqrt(2.0) * sigma_clipped)\n    \n    return tf.reduce_mean(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This will be the perfect score when actual and predicted values are exactly same\nlaplace_log_likelihood(train['FVC'], train['FVC'], 70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using Weeks, Age, Sex and Smoking Status columns from train data\nX = train[['Weeks','Age','Sex','SmokingStatus']].copy()\ny = train['FVC'].copy()\n\n# save the stats for future use\nstats = X.describe().T\n\n# One hot encoding on Sex and SmokingStatus columns\nX = pd.get_dummies(X, columns =['Sex','SmokingStatus'],drop_first=True)\n\n#Scaling numeric features \n# scaling the numeric features\nfor col in ['Weeks', 'Age']:\n    X[col] = (X[col] - stats.loc[col, 'min']) / (stats.loc[col, 'max'] - stats.loc[col, 'min'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import make_scorer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma = 250","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a scorer function\nl1 = (make_scorer(\n    lambda X,y : laplace_log_likelihood(X,y,sigma=sigma).numpy(),\n    greater_is_better=False))\n\ncross_val_score(LinearRegression(),X,y,cv=3,scoring=l1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.copy()\ny = train['FVC'].copy()\n\nX['base_week'] = X.groupby('Patient')['Weeks'].transform('min')\nX['base_FVC'] = X.groupby('Patient')['FVC'].transform('first')\n\n# save the stats for future use\nstats = X.describe().T\n\n# one hot encoding for categorcial features\nX = pd.get_dummies(data=X, columns=['Sex','SmokingStatus'], drop_first=True)\n\n# Scaling numeric columns\nnum_cols = ['Age','Weeks','base_week','base_FVC']\n\n# Min-max scaling\nfor col in num_cols:\n    X[col] = (X[col]-stats.loc[col,'min']) / (stats.loc[col,'max'] - stats.loc[col,'min'])\n    \n# printing the correlation of all features with FVC\nprint(X.corr()['FVC'].abs().sort_values(ascending=False)[1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removing unnecesary columns after transformations\nX.drop(['Patient','Percent','FVC'], axis=1, inplace=True)\nX.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking the score on transformed data now\ncross_val_score(LinearRegression(),X,y,cv=3,scoring=l1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see, the fold scores have improved significantly. Lets fit the Linear model on this data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit on the train dataset\nlr = LinearRegression().fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Making Prediction on Test Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Processing submission file\nsub = sample_sub.Patient_Week.str.extract(\"(ID\\w+)_(\\-?\\d+)\").rename({0: \"Patient\", 1: \"Weeks\"}, axis=1)\nsub['Weeks'] = sub['Weeks'].astype(int)\nsub = pd.merge(sub, test[['Patient', 'Sex', 'SmokingStatus']], on='Patient')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"week_temp = train.groupby([\"Weeks\", 'Sex'])['FVC'].median()\nsex_temp = train.groupby(['Sex'])['FVC'].median()\n\nfor index, week, sex in sub.iloc[:, 1:3].itertuples():\n    if (week, sex) in week_temp:\n        # we assume we are more accurate here\n        sub.loc[index, 'FVC'] = week_temp[week, sex]\n        sub.loc[index, 'Confidence'] = sigma\n    else:\n        # we assume we are less accurate here, boost confidence\n        sub.loc[index, 'FVC'] = sex_temp[sex]\n        sub.loc[index, 'Confidence'] = sigma + 100\n        \nsub.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# swelling confidence as progress in the weeks\nsub[\"Patient_Week\"] = sub.Patient + \"_\" + sub.Weeks.astype(str)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = (sub.drop(['Confidence', 'Patient_Week'], 1)\n     .merge(test[['Patient', 'Weeks', 'FVC', 'Age']], on='Patient')\n     .rename({\"Weeks_y\": \"base_Week\", \"FVC_y\": \"Base_FVC\", \"Weeks_x\": \"Weeks\"}, axis=1)\n     .drop(['Patient', 'FVC_x'], axis=1))\n\n# one hot encoding, We set drop_first as \n# false to ensure the test is same as train\nx = pd.get_dummies(x, columns=['Sex', 'SmokingStatus'])\n\n# # scaling the numeric features\n#for col in ['Weeks', 'Age', 'base_Week', 'Base_FVC']:\n#    x[col] = (x[col] - stats.loc[col, 'min']) / (stats.loc[col, 'max'] - stats.loc[col, 'min'])\n    \nnum_cols = ['Weeks', 'Age', 'base_Week', 'Base_FVC']\nscaler = StandardScaler()\nscaler.fit(x[num_cols])\n\nx = pd.concat([x[['Sex_Male','SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']].reset_index(drop=True),\n                pd.DataFrame(scaler.transform(x[num_cols]),columns=num_cols)],axis=1)\n    \n\nx = x[['Weeks', 'Age', 'base_Week', 'Base_FVC', 'Sex_Male',\n   'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']]\n\nx.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['FVC'] = lr.predict(x)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LR submission\n#sub[['Patient_Week', 'FVC', 'Confidence']].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = train.copy()\n\n# Create base_Week, Base_FVC and Base_Percent for train\ntemp = (x.groupby(\"Patient\")\n        .apply(lambda x: x.loc[int(\n            np.percentile(x['Weeks'].index, q=25)\n        ), [\"Weeks\", \"FVC\", \"Percent\"]]))\n\ntemp.rename(\n    {\"Weeks\": \"Base_Week\", \n     \"FVC\": \"Base_FVC\", \n     \"Percent\": \"Base_Percent\"}, \n    axis=1, inplace=True)\n\n# merge it with train data\nx = x.merge(temp, on='Patient')\nx['Where'] = 'train'\n\n# merge the test dataset as well to be able to handle 1hC\ntemp = sub[['Patient', 'Weeks']].merge(\n    test.rename({\"Weeks\": \"Base_Week\", \n                 \"FVC\": \"Base_FVC\", \n                 \"Percent\": \"Base_Percent\"}, axis=1), \n    on='Patient')\n\n# concatente to the train dataset\ntemp['Where'] = 'test'\nx = pd.concat([x, temp], axis=0)\n\n# create week offsets\nx['Week_Offset'] = x['Weeks'] - x['Base_Week']\n\n# oridinal encode categorical values\nx['Sex'] = x['Sex'].map({\"Male\": 1, \"Female\": 0})\nx['SmokingStatus'] = x['SmokingStatus'].map({\"Ex-smoker\": 0, \"Never smoked\": 1, \"Currently smokes\": 2})\n\n# one hot encoding\nx = pd.get_dummies(x, columns=['Sex', 'SmokingStatus'], drop_first=True)\n\n# binned FVC does better?\nx['Bin_base_FVC'] = pd.cut(x['Base_FVC'], bins=range(0, 7501, 500)).cat.codes / 15\n\n# lets scale the numeric columns (We scale it with max possibe values)\nnum_cols = ['Weeks', 'Week_Offset', 'Base_Week', 'Age', 'Base_FVC', 'Percent', 'Base_Percent']\nfor col in num_cols:\n    x[col] = (x[col] - x[col].min()) / (x[col].max() - x[col].min())\n\nto_drop = (\n    [\"FVC\", 'Percent']\n    \n    + [\n#         \"Base_FVC\", \n#         'Base_Week', \n#         'Weeks', \n#         'Bin_base_FVC', \n#         'Base_Percent'\n    ] + \n    \n    ['Patient']\n)\n\n# print out how well our features would do\nprint (x[x.Where == 'train'].corr()['FVC'].abs().sort_values(ascending=False).drop(to_drop[:-1]))\n\ny = x['FVC'].dropna()\nx = x.drop(to_drop, axis=1)\n\nx.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest, f_regression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import GridSearchCV\n\ngrid_params = {\"SFromModel__k\": range(10)}\n\ntemp = Pipeline(\n    [(\"SFromModel\", SelectKBest(score_func=f_regression)),\n    (\"Model\", LinearRegression())])\n\ngrid = GridSearchCV(temp, param_grid=grid_params, n_jobs=-1, cv=3, scoring=l1)\ngrid.fit(x[x.Where == 'train'].drop('Where', 1), y)\n\nprint (grid.best_params_, grid.best_score_)\nmodel = grid.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score = (0, np.inf, np.inf)\nfor i in range(50, 1500, 50):\n    sigma=i\n    temp = cross_val_score(model, x[x.Where == 'train'].drop('Where', 1), y, cv=3, scoring=l1)\n    if best_score[1] > temp.mean():\n        best_score = i, temp.mean(), temp.std()\n        \nsigma = best_score[0]\nbest_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = LinearRegression().fit(x[x.Where == 'train'].drop('Where', 1), y)\nsub['FVC'] = lr.predict(x[x.Where == 'test'].drop('Where', 1))\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LR submission\nsub['Confidence'] = best_score[0]\nsub[['Patient_Week', 'FVC', 'Confidence']].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}