{"cells":[{"metadata":{},"cell_type":"markdown","source":"*change in version 6: change in proc_df*","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"* [DataFrames](#section-one)\n* [CT Images](#section-two)\n* [LightGBM](#section-three)\n* [Submission](#section-four)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"<a id=\"section-one\"></a>\n# DataFrames","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"First, let's import the necessary modules.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\n\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom pydicom import dcmread\nimport cv2\n\npath = \"/kaggle/input/osic-pulmonary-fibrosis-progression/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, let's have a look at the files:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/osic-pulmonary-fibrosis-progression/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have csv files for train, test datasets, and their image files. Let's get the train.csv, test.csv, and sample_submission.csv:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df  = pd.read_csv(path + \"train.csv\")\ntest_df = pd.read_csv(path + \"test.csv\")\nsubm = pd.read_csv(path + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have 1549 images in the train dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.Patient.nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we have 176 patients in train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.Weeks.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.Weeks.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1)\n\nsns.distplot(train_df[train_df[\"Weeks\"].notna()][\"Weeks\"], ax=ax, color=\"#2222EE\")\nax.set_title(\"distribution of weeks in train\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\"Weeks\" are between -5 and 133, and ths distribution of weeks is as in the above graph.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The FVC value distribution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1)\n\nsns.distplot(train_df[train_df[\"FVC\"].notna()][\"FVC\"], ax=ax, color=\"#22EE22\")\nax.set_title(\"distribution of FVC in train\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Percent value distribution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1)\n\nsns.distplot(train_df[train_df[\"Percent\"].notna()][\"Percent\"], ax=ax, color=\"#EE2222\")\nax.set_title(\"distribution of Percent in train\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Age value distribution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1)\n\nsns.distplot(train_df[train_df[\"Age\"].notna()][\"Age\"], ax=ax, color=\"#992299\")\nax.set_title(\"distribution of Age in train\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sex distribution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.Sex.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.Sex.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(\"Patient\")[\"Sex\"].first().value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Only 21% of the train images are Female.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"SmokingStatus\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"SmokingStatus\"].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, lets have a look at the test dataset:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have only five patients, all male, not any \"currently smokes\", all at age 68 and older. Let's merge submission and test files:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_subm_test(subm,test_df):\n    \n    a = subm['Patient_Week'].str.split(\"_\", expand=True)\n    a.columns=[\"Patient\",\"Week\"]\n        \n    test_df = test_df.merge(a, on=\"Patient\")\n\n    return test_df\n\ntest_df = merge_subm_test(subm,test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.groupby([\"Patient\"])[\"Weeks\"].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.groupby([\"Patient\"])[\"Week\"].first()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.groupby([\"Patient\"])[\"Week\"].last()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have been asked for the estimations of 146 weeks for each patient, starting with -12 and ending with 133.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"<a id=\"section-two\"></a>\n# CT Images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's have a look at a patient's CT scan at the base week (train dataset):","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(5, 6,figsize=(20,20))\nfor n in range(0,30):\n    #print(int(n/6),np.mod(n,6))\n    image = dcmread(\"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/\" + str(n+1) + \".dcm\")\n    axs[int(n/6),np.mod(n,6)].imshow(image.pixel_array);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, et's have a look at a patient's CT scan at the base week, in the test dataset:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(5, 6,figsize=(20,20))\nfor n in range(0,28):\n    #print(int(n/6),np.mod(n,6))\n    image = dcmread(\"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/\" + str(n+1) + \".dcm\")\n    axs[int(n/6),np.mod(n,6)].imshow(image.pixel_array);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id=\"section-three\"></a>\n# LightGBM","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's arrange dataframes for LightGBM:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df  = pd.read_csv(path + \"train.csv\")\ntest_df = pd.read_csv(path + \"test.csv\")\nsubm = pd.read_csv(path + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def proc_df(df):\n    \n    df = pd.concat([df, pd.get_dummies(df[\"SmokingStatus\"], dtype=int)], axis=1)\n    df.drop([\"SmokingStatus\"],axis=1,inplace=True)\n\n    df['Sex'] = df['Sex'].map({'Female': 0, 'Male': 1})\n\n    return df\n\ntrain_df = proc_df(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def proc_train(df):\n\n    df_final = pd.DataFrame()\n\n    for patient, df2 in df.groupby('Patient'):\n        \n        df11 = df2[[\"Patient\",\"Weeks\",\"FVC\"]]\n\n\n        df2 = df2.rename(columns={\n            \"FVC\": \"base_FVC\", \n            \"Percent\": \"base_Percent\", \n            \"Weeks\": \"base_Week\"\n        }, errors=\"raise\")\n        \n        df3 = pd.merge(df11, df2, how='outer', on='Patient')\n        df3 = df3.query('Weeks!=base_Week')\n        df3['week_diff'] = df3['base_Week'] - df3['Weeks']\n\n        df_final = pd.concat([df_final, df3])\n        \n    return df_final.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = proc_train(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = subm['Patient_Week'].str.split(\"_\", expand=True)\na.columns=[\"Patient\",\"Weeks\"]\na[\"Weeks\"] = a[\"Weeks\"].astype(\"int\")\n    \ntest_df.rename(\n    columns={\n        'Weeks': 'base_Week',\n        'FVC': 'base_FVC',\n        'Percent': 'base_Percent',\n        'Age': 'Age'\n    },\n    inplace=True\n)\n\ntest_df = proc_df(test_df)\n\ntest_df = pd.merge(a, test_df, how='left', on=['Patient'])\ntest_df['week_diff'] = test_df['base_Week'] - test_df['Weeks']\n\ntest_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's define the column data types:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(set(train_df.columns)-set(test_df.columns)-{'FVC', 'Percent'}, axis=1)\ntest_df = test_df.drop(set(test_df.columns)-set(train_df.columns), axis=1)\n\nX = train_df.drop([\"Patient\",\"FVC\"], axis=1)\ny = train_df[\"FVC\"]\ntest = test_df.drop([\"Patient\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_fold = 5\n\ndef get_lgbm_model(X_train, y_train, X_val, y_val, fold, param_choice, columns):\n    \n    train_data = lgb.Dataset(X_train, label=y_train)\n    val_data = lgb.Dataset(X_val, label=y_val)\n\n    if param_choice == \"normal\":\n        params = {\n            \"metric\":\"rmse\"\n        }\n    elif param_choice == \"quantile1\":\n        params = {\n            \"objective\":\"quantile\",\n            \"alpha\":0.2,\n            \"metric\":\"quantile\"\n        }    \n    elif param_choice == \"quantile2\":\n        params = {\n            \"objective\":\"quantile\",\n            \"alpha\":0.8,\n            \"metric\":\"quantile\"\n        }    \n    \n    model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)\n    model.fit(\n        X_train, \n        y_train, \n        eval_set=[(X_train, y_train), (X_val, y_val)], \n        verbose=1000, \n        early_stopping_rounds=100\n    )\n\n    fold_importance = pd.DataFrame()\n    print(columns)\n    print(model.feature_importances_)\n    fold_importance[\"feature\"] = columns\n    fold_importance[\"importance\"] = model.feature_importances_\n    #plot_save_feat_imp(fold_importance, fold)    \n    fold_importance = fold_importance.sort_values(by=['importance'])\n    fold_importance.to_csv('feature_importances_'+ y_train.name + \"_\" + param_choice + str(fold) + '.csv')\n    \n    return model\n\ndef get_lgbm_pred(X, y, test, param_choice):\n    print(\"get_lgbm_pred \", param_choice)\n\n    pred = []\n    pred_val = np.zeros((len(X)))\n            \n    #X_train, X_val, y_train, y_val = train_test_split(X_scaled, y.fillna(0).values, test_size=0.2, shuffle=True, random_state=42)\n    kf = KFold(n_splits=num_fold, random_state=None, shuffle=False)\n    fold = 0\n    score = 0\n    for train_index, test_index in kf.split(X, y):\n        fold += 1\n        print(\"fold \", fold)\n    \n        X_train = X.iloc[train_index, :]\n        X_val = X.iloc[test_index, :]\n        y_train = y[train_index]\n        y_val = y[test_index]\n        \n        model = get_lgbm_model(X_train, y_train, X_val, y_val, fold, param_choice, X_train.columns)\n\n\n        if fold ==1:\n            pred.append(model.predict(test))\n        else:\n            pred += model.predict(test)\n\n        \n        pred_val[test_index] = model.predict(X_val)            \n        score = score + np.sqrt(mean_squared_error(y_val, pred_val[test_index]))\n        print(\"score \", str(score/fold))\n            \n                    \n    print(\"\\n\\n\\n\")\n\n    f = open(\"score\",\"a+\")\n    f.write(str(score/num_fold)+\", \")\n    f.close()\n    \n    return pred[0]/num_fold, pred_val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take \"Confidence\" as the difference between 0.1 and 0.9 quantiles:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_FVC_te, pred_FVC_tr = get_lgbm_pred(X, y, test, \"normal\")\n\npred_FVC_te_q1, pred_FVC_tr_q1 = get_lgbm_pred(X, y, test, \"quantile1\")\npred_FVC_te_q2, pred_FVC_tr_q2 = get_lgbm_pred(X, y, test, \"quantile2\")\n\npred_conf_tr = pred_FVC_tr_q2 - pred_FVC_tr_q1\npred_conf_te = pred_FVC_te_q2 - pred_FVC_te_q1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculate the score:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(confidence, fvc, pred_fvc):\n\n    confidence = max(confidence, 70)\n    delta = min(abs(fvc-pred_fvc), 1000)\n    score = -(math.sqrt(2)*(delta/confidence)) - np.log(math.sqrt(2)*confidence)\n        \n    return score\n\ndef calc_score(confidence, fvc, pred_fvc):\n    \n    score = 0\n    for n in range(len(confidence)):\n        score += (metric(confidence[n], fvc[n], pred_fvc[n]))\n        \n    return score/len(train_df)\n\nscore = calc_score(pred_conf_tr, train_df.FVC.values, pred_FVC_tr)\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id=\"section-four\"></a>\n# Submission","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"And finally the submission file:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[\"FVC\"] = pred_FVC_te\nsubm[\"Confidence\"] = pred_conf_te\nsubm.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}