{"cells":[{"metadata":{},"cell_type":"markdown","source":"Another day, another image challenge!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport json\nimport math\nimport cv2\nimport PIL\nfrom PIL import Image\n\nimport plotly.express as px\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom sklearn.decomposition import PCA\nimport os\nimport imagesize\nimport pydicom\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Loading Train and Test Data\n\ntrain = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntest = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\n\nprint(\"{} images in train set.\".format(train.shape[0]))\n\nprint(\"{} images in test set.\".format(test.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Let's look at the distribution of the target","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(train.FVC)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Distribution of the various features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.hist(train['Percent'].values, bins=200)\nplt.title('Histogram Percent counts in train')\nplt.xlabel('Value')\nplt.ylabel('Count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.hist(test['Percent'].values, bins=200)\nplt.title('Histogram Percent counts in test')\nplt.xlabel('Value')\nplt.ylabel('Count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Smoking status","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = train['SmokingStatus'].value_counts().reset_index()\ntt.columns = ['cat', 'status']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.graph_objects as go\n\nlabels = tt.cat.values\nvalues = tt.status.values\n\nfig = go.Figure(data=[go.Pie(labels=labels, values=values)])\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train target distributions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fvc = train.FVC\nfig = px.histogram(fvc)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Distribution of age","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"age = train.Age\nfig = px.histogram(age)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Distribution of sex","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = train['Sex'].value_counts().reset_index()\ntt.columns = ['sex', 'count']\n\nlabels = tt['sex'].values\nvalues = tt['count'].values\nfig = go.Figure(data=[go.Pie(labels=labels, values=values)])\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"DICOM format\n\nDigital Imaging and Communications in Medicine (DICOM) is the accepted standard for the communication and management of medical imaging information. DICOM is used for archiving and transmitting medical images. It enables the integration of medical imaging devices (radiological scanners), servers, network hardware and Picture Archiving and Communication Systems (PACS). The standard was widely adopted by hospitals and research centers and is steadly advancing as well toward small practice and cliniques.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_pixel_array(dataset, figsize=(10,10)):\n    plt.figure(figsize=figsize)\n    plt.imshow(dataset.pixel_array, cmap=plt.cm.bone)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's first take a look at the sample images that are available in train and test sets. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport pydicom\n\ndef show_info(dataset):\n    path = '../input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/'\n    #dataset = pydicom.dcmread(path+'/'+filename)\n    print(\"Patient id..........:\", dataset.PatientID)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)\n            \nfor file_path in glob.glob('../input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/*.dcm'):\n    #print(file_path)\n    filename = file_path.split('/')[-1]\n    dataset = pydicom.dcmread(file_path)\n    show_info(dataset)\n    plot_pixel_array(dataset)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_info(dataset):\n    path = '../input/osic-pulmonary-fibrosis-progression/test/ID00421637202311550012437/'\n    #dataset = pydicom.dcmread(path+'/'+filename)\n    print(\"Patient id..........:\", dataset.PatientID)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)\n\nfor file_path in glob.glob('../input/osic-pulmonary-fibrosis-progression/test/ID00421637202311550012437/*.dcm'):\n    dataset = pydicom.dcmread(file_path)\n    show_info(dataset)\n    plot_pixel_array(dataset)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Train images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_info(dataset):\n    path = '../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/'\n    #dataset = pydicom.dcmread(path+'/'+filename)\n    print(\"Patient id..........:\", dataset.PatientID)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)\n\nfor file_path in glob.glob('../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/*.dcm'):\n    dataset = pydicom.dcmread(file_path)\n    show_info(dataset)\n    plot_pixel_array(dataset)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_info(dataset):\n    path = '../input/osic-pulmonary-fibrosis-progression/train/ID00012637202177665765362/'\n    #dataset = pydicom.dcmread(path+'/'+filename)\n    print(\"Patient id..........:\", dataset.PatientID)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)\n\n\nfor file_path in glob.glob('../input/osic-pulmonary-fibrosis-progression/train/ID00012637202177665765362/*.dcm'):\n    dataset = pydicom.dcmread(file_path)\n    show_info(dataset)\n    plot_pixel_array(dataset)\n    break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import xgboost\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier \n\nimport lightgbm as lgb\nfrom numba import jit ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/train.csv' )\ntest  = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/test.csv' )\n\ntrain['traintest'] = 0\ntest ['traintest'] = 1\nsub   = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )\nsub['Weeks']   = sub['Patient_Week'].apply( lambda x: int(x.split('_')[-1]) )\nsub['Patient'] = sub['Patient_Week'].apply( lambda x: x.split('_')[0] ) \n\ntrain['Sex']           = pd.factorize( train['Sex'] )[0]\ntrain['SmokingStatus'] = pd.factorize( train['SmokingStatus'] )[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Percent']       = (train['Percent'] - train['Percent'].mean()) / train['Percent'].std()\ntrain['Age']           = (train['Age'] - train['Age'].mean()) / train['Age'].std()\ntrain['Sex']           = (train['Sex'] - train['Sex'].mean()) / train['Sex'].std()\ntrain['SmokingStatus'] = (train['SmokingStatus'] - train['SmokingStatus'].mean()) / train['SmokingStatus'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DICT = './'\n\nID = 'Patient_Week'\nTARGET = 'FVC'\nSEED = 42\n\nN_FOLD = 4\n\ntrain = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntrain[ID] = train['Patient'].astype(str) + '_' + train['Weeks'].astype(str)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.DataFrame()\n\ngb = train.groupby('Patient')\ntk0 = tqdm(gb, total=len(gb))\n\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby('Weeks'):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'}\n        tmp = tmp.drop(columns='Patient_Week').rename(columns=rename_cols)\n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']\n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n    \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv').rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'})\n\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\n\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\n\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\n\ntest = submission.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient')\n\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\n\nprint(test.shape)\n\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfolds = train[[ID, 'Patient', TARGET]].copy()\n\nFold = GroupKFold(n_splits=N_FOLD)\n\ngroups = folds['Patient'].values\n\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\n    \nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What is Boosting?\n\n\nBoosting (originally called hypothesis boosting) refers to any Ensemble method that can combine several weak learners into a strong learner. The general idea of most boosting methods is to train predictors sequentially, each trying to correct its predecessor. There are many boosting methods available, but by far the most popular are AdaBoost(short for Adaptive Boosting) and Gradient Boosting. We will talk about both here, but after reading in the data and pre-processing them.\n\n\n![](https://miro.medium.com/max/694/1*QJZ6W-Pck_W7RlIDwUIN9Q.jpeg)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Adaboost\n\n\nOne way for a new predictor to correct its predecessor is to pay a bit more attention to the training instances that the predecessor underfitted. This results in new predictors focusing more and more on the hard cases. This is the technique used by Ada‐Boost. For example, to build an AdaBoost classifier, a first base classifier (such as a Decision Tree) is trained and used to make predictions on the training set. The relative weight of misclassified training instances is then increased. A second classifier is trained using the updated weights and again it makes predictions on the training set, weights are updated, and so on ...\n\n\n![](http://www.github.com/rakash/images1/blob/master/adaboost.jpg?raw=true)\n\nLet us see how decision boundaries are drawn for all the models for adaboost\n\n![](http://www.github.com/rakash/images1/blob/master/adaboost_db.jpg?raw=true)\n\nThe first classifier(notified by the line) gets many instances wrong, so their weights get boosted. The second classifier therefore does a better job on these instances, and so on. The plot on the right represents the same sequence of predictors except that the learning rate is halved (i.e., the misclassified instance weights are boosted half as much at every iteration). As you can see, this sequential learning technique has some similarities with Gradient Descent, except that instead of tweaking a single predictor’s parameters to minimize a cost function, AdaBoost adds predictors to the ensemble,gradually making it better.\n\nOnce all predictors are trained, the ensemble makes predictions very much like bagging or pasting, except that predictors have different weights depending on their overall accuracy on the weighted training set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.tree import DecisionTreeRegressor\n\n\ndef adaboost_it(ada_train_df, ada_test_df):\n    print(\"Ada-Boosting...\")\n    t_splits = 5\n    k_scores = []\n    \n    kf = KFold(n_splits = t_splits)\n    features = [i for i in ada_train_df.columns if i not in ['Patient', 'predict_Week', 'FVC']]\n    target = 'FVC'\n    \n    oof_pred = np.zeros((len(ada_train_df), 4))\n    y_pred = np.zeros((len(ada_test_df), 4))\n    \n    for fold, (tr_ind, val_ind) in enumerate(kf.split(ada_train_df)):\n        print(f'Fold: {fold+1}')\n        x_train, x_val = ada_train_df[features].iloc[tr_ind], ada_train_df[features].iloc[val_ind]\n        y_train, y_val = ada_train_df[target][tr_ind], ada_train_df[target][val_ind]\n        ada_clf = AdaBoostRegressor(DecisionTreeRegressor(random_state=0 ) ) #max_depth=1), n_estimators=200, algorithm=\"SAMME.R\", learning_rate=0.5)\n        ada_clf.fit(x_train, y_train)\n        oof_pred[val_ind] = ada_clf.predict(x_val).reshape(-1, 1)\n        y_pred += ada_clf.predict(ada_test_df[features]).reshape(-1,1) / t_splits\n        \n    y_pred = y_pred.mean(axis=1)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric( trueFVC, predFVC):\n    deltaFVC = np.clip( np.abs(trueFVC-predFVC), 0 , 1000 )  \n    return np.mean( -1*(np.sqrt(2)*deltaFVC/clipSTD) - np.log( np.sqrt(2)*clipSTD ) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(['Sex', 'SmokingStatus', 'Patient', 'Patient_Week'], axis=1, inplace=True)\ntest.drop(['Sex', 'SmokingStatus', 'Patient_Week', 'Patient'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adab_pred = adaboost_it(train, test)\n\ntest['FVC_pred'] = adab_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### XBOOST\n\n\n![](https://miro.medium.com/max/583/1*FLshv-wVDfu-i54OqvZdHg.png)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost\n\ndef xgb(xgb_train_df, xgb_test_df):\n    \n    print(\"XG-Boosting...\")\n    t_splits = 5\n    k_scores = []\n    kf = KFold(n_splits = t_splits)\n    \n    features = [i for i in xgb_train_df.columns if i not in ['Patient', 'predict_Week', 'FVC']]\n    target = 'FVC'\n    oof_pred = np.zeros((len(xgb_train_df), 4))\n    y_pred = np.zeros((len(xgb_test_df), 4))\n    for fold, (tr_ind, val_ind) in enumerate(kf.split(xgb_train_df)):\n        \n        print(f'Fold: {fold+1}')\n        \n        x_train, x_val = xgb_train_df[features].iloc[tr_ind], xgb_train_df[features].iloc[val_ind]\n        y_train, y_val = xgb_train_df[target][tr_ind], xgb_train_df[target][val_ind]\n        \n        xgb_clf = xgboost.XGBRegressor()\n        \n        xgb_clf.fit(x_train, y_train)\n        \n        oof_pred[val_ind] = xgb_clf.predict(x_val).reshape(-1,1)\n      \n        y_pred += xgb_clf.predict(xgb_test_df[features]).reshape(-1,1) / t_splits\n        \n    y_pred = y_pred.mean(axis=1)        \n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_pred = xgb(train, test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Catboost\n\n\nCatboost yields state-of-the-art results without extensive data training typically required by other machine learning methods, and it Provides powerful out-of-the-box support for the more descriptive data formats that accompany many business problems.\n\nMajor advantage is it handles categorical variables automatically, that is why the name 'CAT-boost'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import catboost as cb\n\ndef cat(cat_train_df, cat_test_df):\n    \n    print(\"Meeowwww...\")\n    t_splits = 5\n    k_scores = []\n    kf = KFold(n_splits = t_splits)\n    features = [i for i in cat_train_df.columns if i not in ['Patient', 'predict_Week', 'FVC']]\n    target = 'FVC'\n    oof_pred = np.zeros((len(cat_train_df), 4))\n    y_pred = np.zeros((len(cat_test_df), 4))\n    \n    for fold, (tr_ind, val_ind) in enumerate(kf.split(cat_train_df)):\n        \n        print(f'Fold: {fold+1}')\n        x_train, x_val = cat_train_df[features].iloc[tr_ind], cat_train_df[features].iloc[val_ind]\n        y_train, y_val = cat_train_df[target][tr_ind], cat_train_df[target][val_ind]\n        \n        cat_clf = cb.CatBoostRegressor(logging_level='Silent')\n        \n        cat_clf.fit(x_train, y_train)\n        oof_pred[val_ind] = cat_clf.predict(x_val).reshape(-1,1)\n      \n        y_pred += cat_clf.predict(cat_test_df[features]).reshape(-1,1) / t_splits\n        \n           \n    y_pred = y_pred.mean(axis=1)               \n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_pred = cat(train, test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LightGBM\n\nIt is based on decision tree algorithms, it splits the tree leaf wise with the best fit whereas other boosting algorithms split the tree depth wise or level wise rather than leaf-wise. So when growing on the same leaf in Light GBM, the leaf-wise algorithm can reduce more loss than the level-wise algorithm and hence results in much better accuracy which can rarely be achieved by any of the existing boosting algorithms. Also, it is surprisingly very fast, hence the word ‘Light’.\n\n\n![](https://s3-ap-south-1.amazonaws.com/av-blog-media/wp-content/uploads/2017/06/11194110/leaf.png)\n\n\n\nHOW IT WORKS IN LIGHT GBM\n\n![](https://s3-ap-south-1.amazonaws.com/av-blog-media/wp-content/uploads/2017/06/11194227/depth.png)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\ndef lgbc(lgb_train_df, lgb_test_df):\n    \n    t_splits = 5\n    k_scores = []\n    kf = KFold(n_splits = t_splits)\n    \n    features = [i for i in lgb_train_df.columns if i not in ['Patient', 'predict_Week', 'FVC']]\n    target = 'FVC'\n    \n    oof_pred = np.zeros((len(lgb_train_df), 4))\n    y_pred = np.zeros((len(lgb_test_df), 4))\n    for fold, (tr_ind, val_ind) in enumerate(kf.split(lgb_train_df)):\n        print(f'Fold: {fold+1}')\n        x_train, x_val = lgb_train_df[features].iloc[tr_ind], lgb_train_df[features].iloc[val_ind]\n        y_train, y_val = lgb_train_df[target][tr_ind], lgb_train_df[target][val_ind]\n        \n        lg = lgb.LGBMRegressor(silent=False)\n        lg.fit(x_train, y_train)\n        oof_pred[val_ind] = lg.predict(x_val).reshape(-1,1)\n      \n        y_pred += lg.predict(lgb_test_df[features]).reshape(-1,1) / t_splits\n       \n    y_pred = y_pred.mean(axis=1)        \n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lg_pred = lgbc(train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = lg_pred * 0.25 + xgb_pred * 0.25 + cat_pred * 0.50\nsubmission['FVC'] = sub","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Confidence score -- Thanks to this kernel - https://www.kaggle.com/yasufuminakama/osic-lgb-baseline","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['FVC_pred'] = train.FVC\ntest['FVC_pred'] = sub\n\ntrain['Confidence'] = 100\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy as sp\nfrom functools import partial\n\ndef loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['FVC'] - row['FVC_pred'])\n    delta = min(diff, 1000)\n    score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(train.iterrows(), total=len(train))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    #bounds = [(70, 100)]\n    #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy as sp\nfrom functools import partial\n\ndef loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['base_FVC'] - row['FVC_pred'])\n    delta = min(diff, 1000)\n    score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n    return -score\n\ntresults = []\n\ntk0 = tqdm(test.iterrows(), total=len(test))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    #bounds = [(70, 100)]\n    #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    tresults.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimized score\n\ntrain['Confidence'] = results\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n\nscore = train['score'].mean()\n\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimized score\n\ntest['Confidence'] = tresults\n\ntest['sigma_clipped'] = test['Confidence'].apply(lambda x: max(x, 70))\n\ntest['diff'] = abs(test['base_FVC'] - test['FVC_pred'])\ntest['delta'] = test['diff'].apply(lambda x: min(x, 1000))\ntest['score'] = -math.sqrt(2)*test['delta']/test['sigma_clipped'] - np.log(math.sqrt(2)*test['sigma_clipped'])\n\nscore = test['score'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Confidence'] = tresults\n\nsubmission.to_csv('submission.csv', index=False)\n\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}