{"cells":[{"metadata":{},"cell_type":"markdown","source":"# import necessary dependencies","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# list from directory","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"os.listdir(\"../input/osic-pulmonary-fibrosis-progression\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# read train, test, submission csv","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntest = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# take a look","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# shape of train, test and submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Train info {train.shape}\")\nprint(f\"test info {test.shape}\")\nprint(f\"submission info {submission.shape}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# import visualization packages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# patient column unique id value_counts","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Total Patient Id {train['Patient'].count()}\")\nprint(f\"NUmber of Unique Id {train['Patient'].value_counts().shape[0]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# smoking status bar plot","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"SmokingStatus\"].value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# take a look of .dcm extension","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img = \"../input/osic-pulmonary-fibrosis-progression/train/ID00009637202177434476278/100.dcm\"\nds = pydicom.dcmread(img)\nplt.figure(figsize = (5,5))\nplt.imshow(ds.pixel_array, cmap=plt.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# take a random id and images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\n\n\ndef get_random(smokes):\n    smoke_pat = train[train[\"SmokingStatus\"]==smokes] \n    patientz = [i for i in smoke_pat[\"Patient\"]] # patient id list\n    r_st = random.choice(patientz) # random choice\n    print(r_st)\n    image_dir = f\"../input/osic-pulmonary-fibrosis-progression/train/{r_st}\" # image directory\n    image_list = os.listdir(image_dir) # list of images\n    c = []\n    for t in image_list:\n        first, exts = os.path.splitext(t) # split text\n        first = int(first) # int\n        c.append(first) # append\n    d = [num for num in range(1, 31)] # num from 1 to 30\n    gh = []\n    for x in c:\n        if x in d:\n            gh.append(x) # if number is in list then append\n    fig = plt.figure(figsize=(10, 10)) # figure\n    columns = 5\n    row = 6\n    for ab in gh:\n        files = image_dir + \"/\" + str(ab) + \".dcm\" # file directory\n        ds = pydicom.dcmread(files) # read dcm file\n        fig.add_subplot(row, columns, ab) # add plot\n        plt.imshow(ds.pixel_array, cmap=plt.cm.bone) # show images\n    plt.suptitle(smokes) # title","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# random ex-smoker patient","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"get_random(\"Ex-smoker\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_random(\"Never smoked\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# submission data split and merge with test data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"get_random(\"Currently smokes\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[\"Patient\"] = submission[\"Patient_Week\"].apply(lambda x:x.split(\"_\")[0])\nsubmission[\"Weeks\"] = submission[\"Patient_Week\"].apply(lambda x:x.split(\"_\")[1])\n\nsubmission =  submission[['Patient','Weeks', 'Confidence','Patient_Week']]\nsubmission = submission.merge(test.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# shape of submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# submission data patient unique id","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[\"Patient\"].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# new column","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"Dataset\"] = \"train\"\ntest[\"Dataset\"] = \"test\"\nsubmission[\"Dataset\"] = \"submission\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge test and submission with train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = train.append([test, submission])\ndataset = dataset.reset_index()\ndataset = dataset.drop(columns=['index'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# convert object to int64","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset[\"Weeks\"] = dataset[\"Weeks\"].astype(\"int64\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# dataset information","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# make new column First_week and take min number","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset[\"First_week\"] = dataset[\"Weeks\"]\ndataset.loc[dataset.Dataset=='submission','First_week'] = np.nan\ndataset[\"First_week\"] = dataset.groupby('Patient')['First_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = dataset.merge(dataset[dataset[\"Weeks\"] == dataset[\"First_week\"]][[\"Patient\", \"FVC\"]].rename({\"FVC\": \"First_FVC\"}, axis=1).groupby(\"Patient\").first().reset_index(), on=\"Patient\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# check the week difference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset[\"Week_diff\"] = dataset[\"Weeks\"] - dataset[\"First_week\"]\n# dataset[\"FVC_diff\"] = dataset[\"FVC\"] - dataset[\"First_FVC\"]\n\ndataset = pd.concat([dataset,pd.get_dummies(dataset.Sex),pd.get_dummies(dataset.SmokingStatus)], axis=1)\n\ndataset = dataset.drop(columns=['Sex', 'SmokingStatus'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# split dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = dataset[dataset[\"Dataset\"]==\"train\"]\ntest = dataset[dataset[\"Dataset\"]==\"test\"]\nsubmission = dataset[dataset[\"Dataset\"]==\"submission\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalize data by StandardScalar","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\ncol = ['Weeks', 'Percent', 'Age', 'First_week', 'First_FVC', 'Week_diff',\n       'Female', 'Male', 'Currently smokes', 'Ex-smoker', 'Never smoked']\n\ntrain_data = train[col]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# check null","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Correlation check","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplots(figsize=(14,10))\ng = train.corr()\nsns.heatmap(g, annot=True, fmt='.2', cmap=\"Dark2_r\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# correlation between label(\"FVC\") and others","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"g[\"FVC\"].sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalize","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"stdscale = StandardScaler()\ntrain_data[col] = stdscale.fit_transform(train_data[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[col]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# import necessary packages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.tree import DecisionTreeRegressor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparameter tunning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params = {\n    \"svr\": {\n        \"model\": SVR(gamma=\"auto\"),\n        \"params\": {\n            \"C\": [1, 5, 10, 15, 20],\n            \"kernel\":['linear', 'poly', 'rbf', 'sigmoid']\n        }\n    },\n    \"RandomForest\": {\n        \"model\": RandomForestRegressor(),\n        \"params\": {\n            \"n_estimators\":[100, 200],\n        }\n    },\n    \"LR\": {\n        \"model\": LinearRegression(),\n        \"params\": {\n            \n        }\n    },\n    \"Decision Tree\": {\n        \"model\": DecisionTreeRegressor(),\n        \"params\": {\n            \"splitter\": [\"best\", \"random\"],\n            \"criterion\": [\"mse\", \"mae\"],\n            \"max_depth\": [5, 10, 15],\n        }\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GridSearch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nscores = []\nfor model_name, param in model_params.items():\n    clf = GridSearchCV(param[\"model\"], param[\"params\"], cv=10, return_train_score=False)\n    clf.fit(train_data[col], train[\"FVC\"])\n    scores.append({\n        \"model\": model_name,\n        \"best_score\": clf.best_score_,\n        \"best_params\": clf.best_params_,\n    })\n\ndf = pd.DataFrame(scores, columns=[\"model\", \"best_score\", \"best_params\"])\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# best score given by Linear Regression","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LinearRegression()\n\nmodel.fit(train_data[col], train[\"FVC\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# predict data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(train_data)\npred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# mean squared error and mean absolute error","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error\n\nmse = mean_squared_error(train[\"FVC\"], pred, squared=False)\n\nprint(mse)\n\nmae = mean_absolute_error(train[\"FVC\"], pred)\nprint(mae)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# plot of actual and prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a = list(train[\"FVC\"])\nb = list(pred)\n\nsns.set_style('whitegrid')\nf, ax = plt.subplots(figsize=(15, 5))\nplt.plot(b[:10], c='green', label= 'predictions')\nplt.plot(a[:10], c='red', label= 'actual')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[col].isnull().any()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data = submission[col]\nsub_data = stdscale.fit_transform(sub_data[col])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_2 = model.predict(sub_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# visualize","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a = list(submission[\"FVC\"])\nb = list(pred_2)\n\nsns.set_style('whitegrid')\nf, ax = plt.subplots(figsize=(15, 5))\nplt.plot(b, c='green', label= 'predictions')\nplt.plot(a, c='red', label= 'actual')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# confidence","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[\"FVC_1\"] = pred_2\n\nconfidence_dict={}\nfor id in submission['Patient'].unique():\n    real=float(test[test['Patient']==id]['FVC'])\n    predicted=float(submission[(submission['Patient']==id) & (submission['Weeks'].astype(int)==int(test[test['Patient']==id]['Weeks']))]['FVC_1'])\n    confidence_dict[id]=abs(real-predicted)\n    \n    \nconfidence=[]\nfor i in range(len(submission)):\n    confidence.append(confidence_dict[submission.iloc[i,0]])\nsubmission['Confidence']=confidence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new = submission[[\"Patient_Week\", \"FVC_1\", \"Confidence\"]]\nnew.rename(columns={\"FVC_1\":\"FVC\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# create csv","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"new.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}