{"cells":[{"metadata":{"_uuid":"8cd305bb-94a0-499b-ab2e-6e0694cbadac","_cell_guid":"e2d2dc4a-0d9f-4f12-90f9-ceb5b9291431","trusted":true},"cell_type":"code","source":"import pydicom\nimport os\nfrom os import listdir\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport scipy as sp\nfrom functools import partial\nfrom tqdm.notebook import tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"236acb5d-0d85-4e52-8953-8d7d3538ec26","_cell_guid":"bc87cb68-6f34-41e7-a1bf-7881a3e9d5bd","trusted":true},"cell_type":"code","source":"im_path = \"../input/osic-pulmonary-fibrosis-progressiont/\"\ntrain_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nprint('Training data shape: ', train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7bfce28-786a-436b-b50d-54a1cb932023","_cell_guid":"5a3a9f2c-de98-4122-85d4-1dc930a554e5","trusted":true},"cell_type":"code","source":"# construct train input\ntrain_df['Patient_Week'] = train_df['Patient'].astype(str) + '_' + train_df['Weeks'].astype(str)\noutput = pd.DataFrame()\ngb = train_df.groupby('Patient')\ntk0 = tqdm(gb, total=len(gb))\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby('Weeks'):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'}\n        tmp = tmp.drop(columns='Patient_Week').rename(columns=rename_cols)\n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']\n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n    \ntrain_df = output[output['Week_passed']!=0].reset_index(drop=True)\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75b19b2c-84d1-4d95-b48f-3b4769afebd4","_cell_guid":"15e00f8d-40c4-4257-bb9f-6572f8edd30e","trusted":true},"cell_type":"code","source":"train_df = pd.get_dummies(train_df, columns=['Sex'])\ntrain_df = pd.get_dummies(train_df, columns=['SmokingStatus'])\ntrain_df = train_df.rename(columns={\"Sex_Female\": \"Female\", \n                                    \"Sex_Male\": \"Male\",\n                                    \"SmokingStatus_Currently smokes\": \"CurrentlySmokes\",\n                                    \"SmokingStatus_Ex-smoker\": \"ExSmoker\",\n                                    \"SmokingStatus_Never smoked\": \"NeverSmoked\"})\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f67f4ee-ef05-4411-b6f9-a56f0f410fb1","_cell_guid":"98d336b4-b88f-46c0-ba0a-5ced5a289fa7","trusted":true},"cell_type":"code","source":"X = train_df.drop(['Patient','FVC','base_Week','predict_Week','Patient_Week'], axis=1)\ny = train_df['FVC']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41dc0341-86a8-4126-ab38-a8631205744c","_cell_guid":"6bf9b974-44ab-4155-bde2-17a58e9c904b","trusted":true},"cell_type":"code","source":"# Splite data into training and testing\nfrom sklearn import model_selection\n\n# Reserve 20% for testing\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2, shuffle = False)\n\nprint('training data has ' + str(X_train.shape[0]) + \n      ' observation with ' + str(X_train.shape[1]) + ' features')\nprint('test data has ' + str(X_test.shape[0]) + \n      ' observation with ' + str(X_test.shape[1]) + ' features')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"057faa46-2083-4e9b-a035-02da239e5cef","_cell_guid":"4988816b-6ceb-40bf-a057-7e7819050709","trusted":true},"cell_type":"code","source":"# standardization (x-mean)/std\n# normalization (x-x_min)/(x_max-x_min) ->[0,1]\n\nfrom sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\nscaler.fit(X_train)\nX_train = scaler.transform(X_train)\nX_test = scaler.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e8fc0ed-41a6-4c2d-8c97-1f46a22dd031","_cell_guid":"6d788a96-90fa-49a3-b7e3-82c2854d51ec","trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom xgboost import XGBRegressor\nregr_XGB = XGBRegressor()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cacb7797-0630-4ef3-a18f-3172035902d3","_cell_guid":"400c8f65-fb03-444b-abd8-cbc91faf318e","trusted":true},"cell_type":"code","source":"regr_XGB.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41724a99-3a08-43c4-9b6e-5f8298c0af3b","_cell_guid":"c3310375-64ad-4890-b4ee-46efbb6be9f2","trusted":true},"cell_type":"code","source":"from sklearn import model_selection\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3c0166c-c861-4064-b06a-e1698bc8ec36","_cell_guid":"38ab96e2-fd31-4692-bd29-0cfb32f79871","trusted":true},"cell_type":"code","source":"regr_XGB_opt = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n             colsample_bynode=1, colsample_bytree=0.8999999999999999, eta=0.01,\n             gamma=0, gpu_id=-1, importance_type='gain',\n             interaction_constraints='', learning_rate=0.300000012,\n             max_delta_step=0, max_depth=5, min_child_weight=1, missing=None,\n             monotone_constraints='()', n_estimators=100, n_jobs=0,\n             num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n             scale_pos_weight=1, subsample=0.7999999999999999,\n             tree_method='exact', validate_parameters=1, verbosity=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ca4bb4a-16b2-415e-a04f-69426dc9f2da","_cell_guid":"a06fec19-edf2-4922-9ddd-e11a026ab675","trusted":true},"cell_type":"code","source":"regr_XGB_opt.fit(X_train, y_train)\ny_pred = regr_XGB_opt.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"915d68f1-0c1c-42d4-a7d0-0e4ed0330d1a","_cell_guid":"eea14991-309f-4209-aa63-64b7dd56ab2d","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,5))\nplt.scatter(y_test, y_pred,color = 'r', alpha = 0.3)\nplt.plot([min(y_test),max(y_test)],[min(y_test),max(y_test)], color = 'k')\nplt.xlabel('FVC$_{\\mathrm{test}}$')\nplt.ylabel('FVC$_{\\mathrm{pred}}$')\nplt.rcParams.update({'font.size': 22})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59735084-d432-4832-b745-b8d4a968b980","_cell_guid":"eb0a5348-cca2-45b1-97fe-3d411a2eb6d4","trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nmse = np.sqrt(mean_squared_error(y_test, y_pred))\nprint(\"RMSE: %f\" % (mse**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"989efa6d-5fac-4484-8ad4-8e6dccd75ebe","_cell_guid":"c482a4c9-1590-4cbd-bc30-3b799124afe0","trusted":true},"cell_type":"code","source":"data_dmatrix = xgb.DMatrix(data=X,label=y)\nparams = {\"objective\":\"reg:squarederror\",'colsample_bytree': 0.3,'learning_rate': 0.1,\n                'max_depth': 5, 'alpha': 10}\ncv_results = xgb.cv(dtrain=data_dmatrix, params=params, nfold=10,\n                    num_boost_round=200,early_stopping_rounds=10,metrics=\"rmse\", as_pandas=True, seed=123)\nprint((cv_results[\"test-rmse-mean\"]).tail(1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea62a98b-09b9-4f76-86e7-8bdd6467af7c","_cell_guid":"2281b5d0-67cd-40f8-82df-fce3da1e1ca4","trusted":true},"cell_type":"code","source":"importances = regr_XGB.feature_importances_\nindices = np.argsort(importances)[::-1]\n# Print the feature ranking\nprint(\"Feature importance ranking by XGBoost Model:\")\nfor ind in range(X.shape[1]):\n    print (\"%s : %.4f\" %(X.columns[indices[ind]],importances[indices[ind]]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"210ef409-e1a4-4ac5-9204-b794482c9ed0","_cell_guid":"466f7c40-26f4-459f-b609-5545f066748d","trusted":true},"cell_type":"code","source":"import seaborn as sns\nfig, ax = plt.subplots(figsize=(15,13)) \nsns.heatmap(X.corr(), annot = True, fmt = '.2f')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"892ec189-d405-4767-b275-acec17ca5a3e","_cell_guid":"0597ace9-9e50-4aa4-977d-0f4f6134dd5f","trusted":true},"cell_type":"code","source":"FVC_pred_train = regr_XGB_opt.predict(X_train)\ntrain_df = pd.DataFrame(X_train, columns = X.columns)\ntrain_df['FVC'] = y_train\ntrain_df['FVC_pred'] = FVC_pred_train\n\nFVC_pred_test = regr_XGB_opt.predict(X_test)\ntest_df = pd.DataFrame(X_test, columns = X.columns)\ntest_df['FVC'] = np.asarray(y_test)\ntest_df['FVC_pred'] = FVC_pred_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2492a00-c37e-4e7d-9def-fba19365e9bb","_cell_guid":"81ddabb3-9f82-403b-a4a9-0ee30136e9f5","trusted":true},"cell_type":"code","source":"# baseline score\ntrain_df['Confidence'] = 100\ntrain_df['sigma_clipped'] = train_df['Confidence'].apply(lambda x: max(x, 70))\ntrain_df['diff'] = abs(train_df['FVC'] - train_df['FVC_pred'])\ntrain_df['delta'] = train_df['diff'].apply(lambda x: min(x, 1000))\ntrain_df['score'] = -2**0.5*train_df['delta']/train_df['sigma_clipped'] - np.log(2**0.5*train_df['sigma_clipped'])\nscore = train_df['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f981382-387c-452d-b094-6528d335d782","_cell_guid":"b877d6c4-11bf-4d47-8330-1041cd7bcca7","trusted":true},"cell_type":"code","source":"def loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['FVC'] - row['FVC_pred'])\n    delta = min(diff, 1000)\n    score = -2**0.5*delta/sigma_clipped - np.log(2**0.5*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(test_df.iterrows(), total=len(test_df))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    #bounds = [(70, 100)]\n    #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b5817fe-bb4a-4551-84af-643d8ba32b7c","_cell_guid":"933ba874-071e-4f9d-840e-4c2d7d20f36b","trusted":true},"cell_type":"code","source":"test_df['Confidence'] = results\ntest_df['sigma_clipped'] = test_df['Confidence'].apply(lambda x: max(x, 70))\ntest_df['diff'] = abs(test_df['FVC'] - test_df['FVC_pred'])\ntest_df['delta'] = test_df['diff'].apply(lambda x: min(x, 1000))\ntest_df['score'] = -2**0.5*test_df['delta']/test_df['sigma_clipped'] - np.log(2**0.5*test_df['sigma_clipped'])\nscore = test_df['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc4937f7-ee59-4010-ba1f-cc19a6345e30","_cell_guid":"05ffd5ed-87ed-434c-9258-e48290f79c4a","trusted":true},"cell_type":"code","source":"\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\\\n        .rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'})\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\ntest = submission.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient')\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5087cb45-87ce-4f0c-b1ea-613d2e4cfee3","_cell_guid":"9b618eba-4998-4b0d-950b-13b27bbcd5e9","trusted":true},"cell_type":"code","source":"test = pd.get_dummies(test, columns=['Sex'])\ntest = pd.get_dummies(test, columns=['SmokingStatus'])\ntest = test.rename(columns={\"Sex_Female\": \"Female\", \n                                    \"Sex_Male\": \"Male\",\n                                    \"SmokingStatus_Currently smokes\": \"CurrentlySmokes\",\n                                    \"SmokingStatus_Ex-smoker\": \"ExSmoker\",\n                                    \"SmokingStatus_Never smoked\": \"NeverSmoked\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"490277fe-aa5c-421c-8747-4939443440fc","_cell_guid":"5cb9ac92-8eb9-45a8-a247-eb9a637dce8d","trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f15ffbc5-1476-4db0-b610-0aef050eb34a","_cell_guid":"a234f257-09df-44ae-ba38-dcca81a0cabc","trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ade8ff8c-0954-437e-8dee-a0feeb8b6378","_cell_guid":"db28c038-2173-453e-9e1f-a81966d65373","trusted":true},"cell_type":"code","source":"# sub = submission.drop(columns=['FVC', 'Confidence']).merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], \n#                                                            on='Patient_Week')\nsub = submission.drop(columns=['Patient','predict_Week','FVC', 'Confidence']).merge(test, on='Patient_Week')\n# sub.columns = submission.columns\nsub.to_csv('submission.csv', index=False)\nsub['Female'] = 0\nsub['CurrentlySmokes'] = 0\nsub = sub[['Patient_Week', 'Patient', 'predict_Week', 'base_Week', 'base_FVC', 'base_Percent', 'base_Age', \n           'Week_passed', 'Female', 'Male', 'CurrentlySmokes', 'ExSmoker', 'NeverSmoked']]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"020e9963-0487-4db4-b5f9-ee6f21d95658","_cell_guid":"e6bf4baf-face-41a7-9210-cf5ff6de4cc6","trusted":true},"cell_type":"code","source":"X_test_sub = sub.iloc[:,4:]\nscaler = MinMaxScaler()\nscaler.fit(X_test_sub)\nX_test_sub = scaler.transform(X_test_sub)\nFVC_pred_sub = regr_XGB_opt.predict(X_test_sub)\nsub['FVC_pred'] = FVC_pred_sub \nsub","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"397c854b-37ab-4643-a9c7-41f3b6b07ac0","_cell_guid":"d3d18ff2-84b1-43fc-8cb2-cbcde363c30d","trusted":true},"cell_type":"code","source":"for pid in sub['Patient'].unique():\n# pid = 'ID00426637202313170790466'\n    temp = sub[sub['Patient'] == pid]\n    plt.plot(temp['predict_Week'], temp['FVC_pred'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c868a075-bfb2-4483-a07c-85f32fc38ed2","_cell_guid":"f3fac009-c9a5-48b5-98d6-988bc36c9b5b","trusted":true},"cell_type":"code","source":"attempt1 = submission.merge(sub, on='Patient_Week')\nattempt1 = attempt1.loc[:,['Patient_Week','FVC_pred','Confidence']]\nattempt1.columns = ['Patient_Week','FVC','Confidence']\nattempt1['Confidence'] = 350\nattempt1.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}