{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OSIC - Linear Regression","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\nimport matplotlib.pyplot as plt\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-15T14:00:31.716955Z","iopub.execute_input":"2022-01-15T14:00:31.717505Z","iopub.status.idle":"2022-01-15T14:00:32.609552Z","shell.execute_reply.started":"2022-01-15T14:00:31.717469Z","shell.execute_reply":"2022-01-15T14:00:32.608677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data loading and preprocessing","metadata":{}},{"cell_type":"code","source":"ROOT = Path(\"../input/osic-pulmonary-fibrosis-progression\")\n\ntrain = pd.read_csv(ROOT / 'train.csv')\ntest = pd.read_csv(ROOT / 'test.csv')\nsub = pd.read_csv(ROOT / 'sample_submission.csv')\n\ntrain.drop_duplicates(subset=['Patient', 'Weeks'], keep=False, inplace=True)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:32.663277Z","iopub.execute_input":"2022-01-15T14:00:32.663667Z","iopub.status.idle":"2022-01-15T14:00:32.724470Z","shell.execute_reply.started":"2022-01-15T14:00:32.663637Z","shell.execute_reply":"2022-01-15T14:00:32.723627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:33.304788Z","iopub.execute_input":"2022-01-15T14:00:33.305397Z","iopub.status.idle":"2022-01-15T14:00:33.318862Z","shell.execute_reply.started":"2022-01-15T14:00:33.305345Z","shell.execute_reply":"2022-01-15T14:00:33.317711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create training data\ntrainData = []\nfor p in train['Patient'].unique():\n    patientData = train[train['Patient'] == p]\n    firstMeasure = list(patientData.iloc[0, :].values)\n        \n    for i, week in enumerate(patientData['Weeks'].iloc[1:]):\n        fvc = patientData.iloc[i, 2]\n        trainDataPoint = firstMeasure + [week, fvc]\n        trainData.append(trainDataPoint)\n    \n        \ntrainData = pd.DataFrame(trainData)\n\ntrainData.columns = ['PatientID', 'first_week', 'first_FVC', 'first_Percent', 'Age', 'Sex', 'SmokingStatus'] + ['target_week', 'target_FVC']\n#trainData['delta_week'] = trainData['target_week'] - trainData['first_week']\ntrainData.drop(columns = ['first_Percent'], inplace = True)#, 'target_week', 'first_week'], inplace = True)\n\ntrainData.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:33.841332Z","iopub.execute_input":"2022-01-15T14:00:33.841929Z","iopub.status.idle":"2022-01-15T14:00:34.059118Z","shell.execute_reply.started":"2022-01-15T14:00:33.841894Z","shell.execute_reply":"2022-01-15T14:00:34.057833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create testing data\nsubSplit = np.array(list(sub['Patient_Week'].apply(lambda x: x.split('_')).values))\ntestData = []\nfor p in np.unique(subSplit[:, 0]):\n    patientData = test[test['Patient'] == p]\n    firstMeasure = list(patientData.iloc[0, :].values)\n    for week in subSplit[subSplit[:, 0] == p, 1]:\n        testDataPoint = firstMeasure + [week]\n        testData.append(testDataPoint)\ntestData = pd.DataFrame(testData)\ntestData.columns = ['PatientID', 'first_week', 'first_FVC', 'first_Percent', 'Age', 'Sex', 'SmokingStatus'] + ['target_week']\n\n#testData['delta_week'] = testData['target_week'].map(int) - testData['first_week']\ntestData.drop(columns = ['first_Percent'], inplace = True)#, 'first_week'], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:35.398895Z","iopub.execute_input":"2022-01-15T14:00:35.399398Z","iopub.status.idle":"2022-01-15T14:00:35.422631Z","shell.execute_reply.started":"2022-01-15T14:00:35.399335Z","shell.execute_reply":"2022-01-15T14:00:35.421659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fe engineering\n# trainData.drop(columns = ['PatientID'], inplace = True)\n# testData.drop(columns = ['PatientID'], inplace = True)\n\nle = LabelEncoder()\n\ntrainData['Sex'] = le.fit_transform(trainData['Sex'])\ntestData['Sex'] = le.transform(testData['Sex'])\n\ntrainData['SmokingStatus'] = le.fit_transform(trainData['SmokingStatus'])\ntestData['SmokingStatus'] = le.transform(testData['SmokingStatus'])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:35.850479Z","iopub.execute_input":"2022-01-15T14:00:35.850841Z","iopub.status.idle":"2022-01-15T14:00:35.859460Z","shell.execute_reply.started":"2022-01-15T14:00:35.850808Z","shell.execute_reply":"2022-01-15T14:00:35.858498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.neighbors import KNeighborsRegressor\n\nmodel = LinearRegression()\nmodel.fit(trainData.drop(columns = ['PatientID', 'target_FVC']), trainData['target_FVC'])\nprediction = model.predict(testData.drop(columns = ['PatientID']))#, 'target_week']))","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:36.334780Z","iopub.execute_input":"2022-01-15T14:00:36.335480Z","iopub.status.idle":"2022-01-15T14:00:36.612771Z","shell.execute_reply.started":"2022-01-15T14:00:36.335440Z","shell.execute_reply":"2022-01-15T14:00:36.611940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = []\nfor i in range(testData.shape[0]):\n    patient, week, pred = testData.loc[i, 'PatientID'], testData.loc[i, 'target_week'], prediction[i]\n    confidence = np.random.uniform(0, 300)\n    sub.append([patient + '_' + str(week), pred, confidence])\nsub = pd.DataFrame(sub)\nsub.columns = ['Patient_Week', 'FVC', 'Confidence']","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:07:09.015646Z","iopub.execute_input":"2022-01-15T14:07:09.016006Z","iopub.status.idle":"2022-01-15T14:07:09.044232Z","shell.execute_reply.started":"2022-01-15T14:07:09.015971Z","shell.execute_reply":"2022-01-15T14:07:09.043241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:37.744048Z","iopub.execute_input":"2022-01-15T14:00:37.744622Z","iopub.status.idle":"2022-01-15T14:00:38.317384Z","shell.execute_reply.started":"2022-01-15T14:00:37.744572Z","shell.execute_reply":"2022-01-15T14:00:38.316613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:38.585290Z","iopub.execute_input":"2022-01-15T14:00:38.585921Z","iopub.status.idle":"2022-01-15T14:00:38.599854Z","shell.execute_reply.started":"2022-01-15T14:00:38.585873Z","shell.execute_reply":"2022-01-15T14:00:38.598969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cheat_test = sub.copy()\ncheat_test","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:40.900006Z","iopub.execute_input":"2022-01-15T14:00:40.900333Z","iopub.status.idle":"2022-01-15T14:00:40.913662Z","shell.execute_reply.started":"2022-01-15T14:00:40.900305Z","shell.execute_reply":"2022-01-15T14:00:40.912818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cheat_test[['Patient','Week']] = cheat_test['Patient_Week'].str.split('_',expand=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:48.789212Z","iopub.execute_input":"2022-01-15T14:00:48.789603Z","iopub.status.idle":"2022-01-15T14:00:48.800711Z","shell.execute_reply.started":"2022-01-15T14:00:48.789570Z","shell.execute_reply":"2022-01-15T14:00:48.799707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cheat_test['Week'] = cheat_test['Week'].apply(pd.to_numeric)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:49.430730Z","iopub.execute_input":"2022-01-15T14:00:49.431053Z","iopub.status.idle":"2022-01-15T14:00:49.451827Z","shell.execute_reply.started":"2022-01-15T14:00:49.431026Z","shell.execute_reply":"2022-01-15T14:00:49.450832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cheat_test[(cheat_test.Patient == 'ID00419637202311204720264') & (cheat_test.Week.isin([6, ]))]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:00:50.293510Z","iopub.execute_input":"2022-01-15T14:00:50.293837Z","iopub.status.idle":"2022-01-15T14:00:50.307260Z","shell.execute_reply.started":"2022-01-15T14:00:50.293809Z","shell.execute_reply":"2022-01-15T14:00:50.306120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = []\n\nfor patient in test['Patient'].unique():\n    \n    print(patient)\n    \n    true_weeks = train[train.Patient == patient].Weeks.values\n    true_fvc = train[train.Patient == patient].FVC.values\n        \n    cheat_fvc = cheat_test[(cheat_test.Patient == patient) & (cheat_test.Week.isin(true_weeks))].FVC.values\n    \n    avg.append(abs(sum(cheat_fvc - true_fvc)/len(cheat_fvc)))\n","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:03:41.729829Z","iopub.execute_input":"2022-01-15T14:03:41.730182Z","iopub.status.idle":"2022-01-15T14:03:41.752667Z","shell.execute_reply.started":"2022-01-15T14:03:41.730153Z","shell.execute_reply":"2022-01-15T14:03:41.751469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = np.array(avg)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:03:50.954488Z","iopub.execute_input":"2022-01-15T14:03:50.955170Z","iopub.status.idle":"2022-01-15T14:03:50.959796Z","shell.execute_reply.started":"2022-01-15T14:03:50.955120Z","shell.execute_reply":"2022-01-15T14:03:50.958848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg.mean()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:03:57.749077Z","iopub.execute_input":"2022-01-15T14:03:57.749563Z","iopub.status.idle":"2022-01-15T14:03:57.754919Z","shell.execute_reply.started":"2022-01-15T14:03:57.749531Z","shell.execute_reply":"2022-01-15T14:03:57.754200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg.std()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:04:01.830603Z","iopub.execute_input":"2022-01-15T14:04:01.831227Z","iopub.status.idle":"2022-01-15T14:04:01.837944Z","shell.execute_reply.started":"2022-01-15T14:04:01.831177Z","shell.execute_reply":"2022-01-15T14:04:01.837174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg","metadata":{"execution":{"iopub.status.busy":"2022-01-15T14:04:09.006780Z","iopub.execute_input":"2022-01-15T14:04:09.007127Z","iopub.status.idle":"2022-01-15T14:04:09.012307Z","shell.execute_reply.started":"2022-01-15T14:04:09.007098Z","shell.execute_reply":"2022-01-15T14:04:09.011650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}