{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Predict the rate of decline of FVC & forecast the final 3 FVC with the confidence","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Understanding the data\n\n## 1)Train data\n### i)Baseline FVC\n### ii)Baseline CT Scan\n### iii)Time Series of FVC \n\n## 1)Test data\n### i)Baseline FVC\n### ii)Baseline CT Scan\n\n## Forecast the final 3 FVC with confidence","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lets load & have look at the train and test data\ntrain_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#LETS SORT THE PATIENT-ID FOLLOWED BY WEEKS\nsort_train=train_df.sort_values([\"Patient\",\"Weeks\"])\nsort_test=test_df.sort_values([\"Patient\",\"Weeks\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets get a basic knack of the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape of Training data: ', train_df.shape)\nprint('Shape of Test data: ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check for missing values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### No missing values :) !!!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Number of unique ids are {train_df['Patient'].value_counts().shape[0]} \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patient_ids = set(train_df['Patient'].unique())\ntest_patient_ids = set(test_df['Patient'].unique())\n\ntrain_patient_ids.intersection(test_patient_ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The test set contains patient from the train set So we have more info about the 5 patients than just the baseline FVC","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = train_df.keys()\ncolumns = list(columns)\nprint(columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom os import listdir\nimport pandas as pd\nimport numpy as np\nimport glob\nimport tqdm\nfrom typing import Dict\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n#color\nfrom colorama import Fore, Back, Style\n\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n\n#pydicom\nimport pydicom","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check out number of dcm images,a CT Scan involves multiple images for different cross-sections","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"files = folders = 0\n\npath = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train\"\n\nfor _, dirnames, filenames in os.walk(path):\n  # ^ this idiom means \"we won't be using this value\"\n    files += len(filenames)\n    folders += len(dirnames)\n\nprint(Fore.YELLOW +f'{files:,}',Style.RESET_ALL,\"files/images, \" + Fore.BLUE + f'{folders:,}',Style.RESET_ALL ,'folders/patients')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## So,CT Scan contains about 200 images and each .dcm file is 0.5MB\n## So,100MB/patient and total about 20GB ,now it makes sense","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Lets analyse the distribution of various features and their correlation with decay in FVC","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Lets make a new df containing only unique patient details excluding FVC and week","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_df = train_df.groupby([train_df.Patient,train_df.Age,train_df.Sex, train_df.SmokingStatus])['Patient'].count()\npatient_df.index = patient_df.index.set_names(['PatientId','Age','Sex','SmokingStatus'])\npatient_df = patient_df.reset_index()\npatient_df.rename(columns = {'Patient': 'freq'},inplace = True)\npatient_df.rename(columns = {'PatientId': 'Patient'},inplace = True)\npatient_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(patient_df[\"freq\"],bins=5,color='green')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(patient_df[\"Age\"],bins=20,color='blue')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(patient_df[\"SmokingStatus\"],color=\"orange\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(patient_df[\"Sex\"],color=\"red\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objs as go\n\nfig = px.histogram(patient_df, x='SmokingStatus',color = 'Sex')\nfig.update_traces(marker_line_color='black',marker_line_width=2, opacity=0.85)\nfig.update_layout(title = 'Distribution of SmokingStatus for unique patients')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(patient_df, x='Age',color = 'Sex')\nfig.update_layout(title = 'Distribution of Age w.r.t Sex for unique patients')\nfig.update_traces(marker_line_color='black',marker_line_width=1.5, opacity=0.85)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(patient_df, x='Age',color = 'SmokingStatus')\nfig.update_layout(title = 'Distribution of Age w.r.t SmokingStatus for unique patients')\nfig.update_traces(marker_line_color='black',marker_line_width=1.5, opacity=0.85)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\ntrain_df['Weeks'].value_counts().iplot(kind='barh',\n                                      xTitle='Counts(Weeks)', \n                                      linecolor='black', \n                                      opacity=0.8,\n                                      color='violet',\n                                      theme='pearl',\n                                      bargap=0.2,\n                                      gridcolor='black',\n                                      title='Distribution of the Weeks in the training set')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['FVC'].iplot(kind='hist',\n                      xTitle='Lung Capacity(ml)', \n                      linecolor='black', \n                      opacity=0.8,\n                      color='orange',\n                      bargap=0.5,\n                      gridcolor='white',\n                      title='Distribution of the FVC in the training set')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train_df, x=\"FVC\", y=\"Percent\", color='Age')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train_df, x=\"FVC\", y=\"Age\", color='Sex')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train_df, x=\"FVC\", y=\"Weeks\", color='SmokingStatus')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now lets visualize the time series data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient1 = train_df[train_df.Patient == 'ID00007637202177411956430']\npatient2 = train_df[train_df.Patient == 'ID00012637202177665765362']\npatient3 = train_df[train_df.Patient == 'ID00082637202201836229724']\npatient4 = train_df[train_df.Patient == 'ID00011637202177653955184']\n\npatient1['text'] ='ID: ' + (patient1['Patient']).astype(str) + '<br>FVC ' + patient1['FVC'].astype(str) + '<br>Percent ' + patient1['Percent'].astype(str) + '<br>Week ' + patient1['Weeks'].astype(str)\npatient2['text'] ='ID: ' + (patient2['Patient']).astype(str) + '<br>FVC ' + patient2['FVC'].astype(str)+ '<br>Percent ' + patient2['Percent'].astype(str)  + '<br>Week ' + patient2['Weeks'].astype(str)\npatient3['text'] ='ID: ' + (patient3['Patient']).astype(str) + '<br>FVC ' + patient3['FVC'].astype(str) + '<br>Percent ' + patient3['Percent'].astype(str) + '<br>Week ' + patient3['Weeks'].astype(str)\n\nfig = go.Figure()\nfig.add_trace(go.Scatter(x=patient1['Weeks'], y=patient1['FVC'],hovertext = patient1['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2),\n                    name='Ex-smoker'))\nfig.add_trace(go.Scatter(x=patient2['Weeks'], y=patient2['FVC'],hovertext = patient2['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2),\n                    name='Never smoked'))\nfig.add_trace(go.Scatter(x=patient3['Weeks'], y=patient3['FVC'],hovertext = patient3['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2), name='Currently smokes'))\n\nfig.update(layout_title_text='FVC vs Weeks for different patients')\nfig.update_layout( width=1000,height=700)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient1['text'] ='ID: ' + (patient1['Patient']).astype(str) + '<br>Percent ' + patient1['Percent'].astype(str) + '<br>FVC ' + patient1['FVC'].astype(str) + '<br>Week ' + patient1['Weeks'].astype(str)\npatient2['text'] ='ID: ' + (patient2['Patient']).astype(str) + '<br>Percent ' + patient2['Percent'].astype(str) + '<br>FVC ' + patient2['FVC'].astype(str) + '<br>Week ' + patient2['Weeks'].astype(str)\npatient3['text'] ='ID: ' + (patient3['Patient']).astype(str) + '<br>Percent ' + patient3['Percent'].astype(str) + '<br>FVC ' + patient3['FVC'].astype(str) + '<br>Week ' + patient3['Weeks'].astype(str)\n\n\nfig = go.Figure()\nfig.add_trace(go.Scatter(x=patient1['Weeks'], y=patient1['Percent'],hovertext = patient1['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2),\n                    name='Ex-smoker'))\nfig.add_trace(go.Scatter(x=patient2['Weeks'], y=patient2['Percent'],hovertext = patient2['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2),\n                    name='Never smoked'))\nfig.add_trace(go.Scatter(x=patient3['Weeks'], y=patient3['Percent'],hovertext = patient3['text'],\n                    mode='lines+markers',marker=dict(size = 12,line_width = 2), name='Currently smokes'))\n\nfig.update(layout_title_text='Percent vs Weeks for 3 different patients')\nfig.update_layout( width=700,height=500)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets plot time-Series for a lot of patients to get a general idea","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=px.line(train_df.loc[650:800,:],x=\"Weeks\",y=\"FVC\",color=\"Sex\",line_group=\"Patient\",hover_name=\"Patient\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=px.line(train_df.loc[650:800,:],x=\"Weeks\",y=\"Percent\",color=\"Sex\",line_group=\"Patient\",hover_name=\"Patient\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=px.line(train_df.loc[650:800,:],x=\"Weeks\",y=\"FVC\",color=\"SmokingStatus\",line_group=\"Patient\",hover_name=\"Patient\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=px.line(train_df.loc[650:800,:],x=\"Weeks\",y=\"Percent\",color=\"SmokingStatus\",line_group=\"Patient\",hover_name=\"Patient\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets try implementing a very naive model w/o considering the CT Scan\n## We will try predicting interpolating the points we have in the train data irrespective of all features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"output=pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n## This file has 730 rows i.e.5*146 where 146 the number of weeks -12 to 133\n#And 3 col\noutput.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_patients=test_df[\"Patient\"].values.tolist()\ntest_patients","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make a dictionary of available time-Series data for each test patient","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_dict={patient :{} for patient in test_patients}\npatient_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for patient in patient_dict.keys():\n    for i in range(len(test_df)):\n        if(test_df.loc[i,\"Patient\"]==patient):\n            patient_dict[patient][test_df.loc[i,\"Weeks\"]]=test_df.loc[i,\"FVC\"]\n    for i in range(len(train_df)):\n        if(train_df.loc[i,\"Patient\"]==patient):\n            patient_dict[patient][train_df.loc[i,\"Weeks\"]]=train_df.loc[i,\"FVC\"]\n\nprint(patient_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.interpolate import interp1d\n\nfor patient in patient_dict.keys():\n    x=list(patient_dict[patient].keys())\n    print(x)\n    y=list(patient_dict[patient].values())\n    print(y)\n    plt.scatter(x,y)\n    f=interp1d(x,y,fill_value='extrapolate')\n    x_test=np.arange(-12,134)\n    y_test=f(x_test)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets make a submission with a confidence of 100 ml itself as it is a pretty crude model so no big expectations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(output)):\n    patient=output.loc[i,\"Patient_Week\"][:25]\n    x=list(patient_dict[patient].keys())\n    y=list(patient_dict[patient].values())\n    f=interp1d(x,y,fill_value='extrapolate')\n    temp=max(min(f(int(output.loc[i,\"Patient_Week\"][26:])),1.1*y[0]),0.85*y[0])\n    output.loc[i,\"FVC\"]=temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Really Bad -13 i.e. about 500 ml average deviation\n## I guess using the baseline FVC each time would have done better bcoz interpolating fairly away from data pts isnt sound\n## Lets try that as well","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"length=len(\"ID00419637202311204720264\")\nfor i in range(len(output)):\n    patient=output.loc[i,\"Patient_Week\"][:25]\n    x=list(patient_dict[patient].keys())\n    y=list(patient_dict[patient].values())\n    temp=y[0]\n    output.loc[i,\"FVC\"]=temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fairly good 8.12 :),Pity those who overfit complex models and got a higher laplacian log likelihood","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Now lets take a step forward and model the fluctuation but still w/o CT Scan\n## Bcoz there's still a lot to improve before moving on to image features","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Another trivial model would be weighted average of the data pts\n## Lets try that before moving to regression","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"length=len(\"ID00419637202311204720264\")\nfor i in range(len(output)):\n    patient=output.loc[i,\"Patient_Week\"][:25]\n    x=list(patient_dict[patient].keys())\n    y=list(patient_dict[patient].values())\n    temp=(y[-1]+y[-2]*0.9+y[-3]*0.81+y[-4]*0.72+y[-5]*0.64+y[-6]*0.56)/(1+0.9+0.81+0.72+0.64+0.56)\n    output.loc[i,\"FVC\"]=temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## No significant improvement","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_0=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_1=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_2=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_3=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_4=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_5=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_6=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_7=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_8=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_9=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\ntest_df_10=pd.DataFrame(columns=[\"Patient\",\"Weeks\",\"FVC\",\"Percent\",\"Age\",\"Sex\",\"SmokingStatus\"])\n\ni=0\nfor patient in test_df.Patient:\n    j=0\n    for k in range(len(train_df)):\n        if(train_df.loc[k,\"Patient\"]==patient):\n            eval(\"test_df_\" + str(j)).loc[i,:]=train_df.loc[k,:]\n            j+=1\n    i+=1\n\ntest_df_5.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_4.head(50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets try ElasticNet regression hopefully we will tch the 7 mark","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\n\ntrain = pd.concat([train_df,test_df])\n\noutput = pd.DataFrame()\n\ntrain_uniq = train.groupby('Patient') # Combines all col data by object name and return mean values respectively\n\ntk0 = tqdm(train_uniq, total = len(train_uniq))\n\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby(\"Weeks\"):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}\n        \n        tmp = tmp.rename(columns = rename_cols)\n        \n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent'] \n        \n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        \n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        \n        # Concat the empty DF with edited DF\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n        \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test_df.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})\n\n# Adding Sample Submission\nsubmission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n\n# In submisison file, format: ID_'week', using lambda to split the ID\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])\n\n# In submisison file, format: ID_'week', using lambda to split the Week\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\n\ntest = submission.drop(columns = [\"FVC\", \"Confidence\"]).merge(test, on = 'Patient')\n\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\n\ntest.set_index('Patient_Week', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):\n    trn_idx = folds[folds.fold!=fold_num].index\n    val_idx = folds[folds.fold==fold_num].index\n    \n    y_tr = target.iloc[trn_idx].values\n    X_tr = train_df.iloc[trn_idx][features].values\n    y_val = target.iloc[val_idx].values\n    X_val = train_df.iloc[val_idx][features].values\n    \n    oof = np.zeros(len(train_df))\n    predictions = np.zeros(len(test_df))\n    clf.fit(X_tr, y_tr)\n    \n    oof[val_idx] = clf.predict(X_val)\n    predictions += clf.predict(test_df[features])\n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_kfold_model(clf, train, test, folds, features, target, n_fold=9):\n    \n    # n_fold from 5 to 7\n    \n    oof = np.zeros(len(train))\n    predictions = np.zeros(len(test))\n    feature_importance_df = pd.DataFrame()\n\n    for fold_ in range(n_fold):\n\n        _oof, _predictions = run_single_model(clf,train, test, folds, features, target, fold_num = fold_)\n\n        oof += _oof\n        predictions += _predictions/n_fold\n    \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfrom sklearn.linear_model import Ridge, ElasticNet\n\nTARGET='FVC'\nN_FOLD=9\nfolds = train[['Patient', TARGET]].copy()\nfolds = train[['Patient', TARGET]].copy()\nFold = GroupKFold(n_splits=N_FOLD)\ngroups = folds['Patient'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train[TARGET]\ntest[TARGET] = np.nan # Displays all Null values\n# features\ncat_features = ['Sex', 'SmokingStatus'] # Categorical Features\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)] # Numerical Features\n\nfeatures = num_features + cat_features\ndrop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']\nfeatures = [c for c in features if c not in drop_features]\n\nif cat_features:\n    ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')\n    ce_oe.fit(train)\n    train = ce_oe.transform(train)\n    test = ce_oe.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nfrom functools import partial\nimport scipy as sp\n\nfor alpha1 in [0.3]:\n    for l1s in [0.8]:\n        \n        print(\" For alpha:\",alpha1,\"& l1_ratio:\",l1s)\n        clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)\n        oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\n        train['FVC_pred'] = oof\n        test['FVC_pred'] = predictions\n        # baseline score\n        train['Confidence'] = 100\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)\n\n        def loss_func(weight, row):\n            confidence = weight\n            sigma_clipped = max(confidence, 70)\n            diff = abs(row['FVC'] - row['FVC_pred'])\n            delta = min(diff, 1000)\n            score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n            return -score\n\n        results = []\n        tk0 = tqdm(train.iterrows(), total=len(train))\n        for _, row in tk0:\n            loss_partial = partial(loss_func, row=row)\n            weight = [100]\n            result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n            x = result['x']\n            results.append(x[0])\n\n        # optimized score\n        train['Confidence'] = results\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'Confidence'\n\ntarget = train[TARGET]\ntest[TARGET] = np.nan\nID=\"Patient_Week\"\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']\nfeatures = [c for c in features if c not in drop_features]\n\noof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\ntrain['Confidence'] = oof\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['Confidence'] = predictions\ntest = test.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.tail(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')\nsub = sub.rename(columns={'FVC_pred': 'FVC'})\n\nfor i in range(len(test_df)):\n    sub.loc[sub['Patient_Week']==test_df.Patient[i]+'_'+str(test_df.Weeks[i]), 'FVC'] = test_df.FVC[i]\n    sub.loc[sub['Patient_Week']==test_df.Patient[i]+'_'+str(test_df.Weeks[i]), 'Confidence'] = 0.1\n    \nsub[sub.Confidence<1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(sub)):\n    sub.loc[i,\"Confidence\"]=150\n\nsub.to_csv('submission.csv', index=False, float_format='%.1f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}