{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport os\nimport random\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import raw data and copy\ntrain = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ID='Patient_Week'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Take mean/first of several tests on same weeks\ntrain_temp = train.groupby(['Patient', 'Weeks']).mean()[['FVC', 'Percent']]\ntrain_temp[['Age', 'Sex', 'SmokingStatus']] = train.groupby(['Patient', 'Weeks']).first()[['Age', 'Sex', 'SmokingStatus']]\ntrain = train_temp.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Min_week'] = train.groupby('Patient')['Weeks'].transform('min')\nbase = train.loc[train.Weeks == train.Min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','Base_FVC']\ntrain = train.merge(base, on='Patient', how='left')\ntrain['Base_week'] = train['Weeks'] - train['Min_week']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #Data Augmentation\n# rename_cols = {'Weeks_x':'Base_week', 'FVC_x': 'Base_FVC', 'Percent_x': 'Base_percent', 'Age_x': 'Age', 'SmokingStatus_x': 'SmokingStatus', 'Sex_x':'Sex', 'Weeks_y':'Weeks', 'FVC_y': 'FVC'}\n# drop_cols = ['Age_y', 'Sex_y', 'SmokingStatus_y', 'Percent_y']\n# train = train.merge(train, how='left', left_on='Patient', right_on='Patient').rename(columns=rename_cols).drop(columns=drop_cols)\n# train[ID] = train['Patient'].astype(str) + '_' + train['Weeks'].astype(str)\n# train = train[['Patient', 'Base_week', 'Base_FVC', 'Base_percent', 'Age', 'Sex', 'SmokingStatus', 'Weeks', 'Patient_Week', 'FVC']].reset_index(drop=True)\n# #Keep base week atm\n# # train = train[train['Weeks']!=train['Base_week']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder as SklearnOneHotEncoder\n\n#Override OneHotEncoder to have the column names created automatically (Ex-smoker, Never Smoked...) \nclass OneHotEncoder(SklearnOneHotEncoder):\n    def __init__(self, **kwargs):\n        super(OneHotEncoder, self).__init__(**kwargs)\n        self.fit_flag = False\n\n    def fit(self, X, **kwargs):\n        out = super().fit(X)\n        self.fit_flag = True\n        return out\n\n    def transform(self, X, categories, index='', name='', **kwargs):\n        sparse_matrix = super(OneHotEncoder, self).transform(X)\n        new_columns = self.get_new_columns(X=X, name=name, categories=categories)\n        d_out = pd.DataFrame(sparse_matrix.toarray(), columns=new_columns, index=index)\n        return d_out\n\n    def fit_transform(self, X, categories, index, name, **kwargs):\n        self.fit(X)\n        return self.transform(X, categories=categories, index=index, name=name)\n\n    def get_new_columns(self, X, name, categories):\n        new_columns = []\n        for j in range(len(categories)):\n            new_columns.append('{}_{}'.format(name, categories[j]))\n        return new_columns\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.exceptions import NotFittedError\n\ndef standardisation(x, u, s):\n    return (x-u)/s\n\ndef normalization(x, ma, mi):\n    return (x-mi)/(ma-mi)\n\nclass data_preparation():\n    def __init__(self, bool_normalization=True,bool_standard=False):\n        self.enc_sex = LabelEncoder()\n        self.enc_smok = LabelEncoder()\n        self.onehotenc_smok = OneHotEncoder()\n        self.standardisation = bool_standard\n        self.normalization = bool_normalization\n        \n        \n    def __call__(self, data_untransformed):\n        data = data_untransformed.copy(deep=True)\n        \n        #For the test set/Already fitted\n        try:\n            data['Sex'] = self.enc_sex.transform(data['Sex'].values)\n            data['SmokingStatus'] = self.enc_smok.transform(data['SmokingStatus'].values)\n            data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index).astype(int)], axis=1)\n            \n            #Standardisation\n            if self.standardisation:\n                data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)\n                data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)\n                data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)\n                data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)\n                data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)\n            \n            #Normalization\n            if self.normalization:\n                data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)\n                data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)\n                data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)\n                data['Age'] = normalization(data['Age'],self.age_max,self.age_min)\n                data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)\n                data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)\n\n        #For the train set/Not yet fitted    \n        except NotFittedError:\n            data['Sex'] = self.enc_sex.fit_transform(data['Sex'].values)\n            data['SmokingStatus'] = self.enc_smok.fit_transform(data['SmokingStatus'].values)\n            data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.fit_transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index).astype(int)], axis=1)\n            \n            #Standardisation\n            if self.standardisation:\n                self.base_week_mean = data['Base_week'].mean()\n                self.base_week_std = data['Base_week'].std()\n                data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)\n\n                self.base_fvc_mean = data['Base_FVC'].mean()\n                self.base_fvc_std = data['Base_FVC'].std()\n                data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)\n\n                self.base_percent_mean = data['Base_percent'].mean()\n                self.base_percent_std = data['Base_percent'].std()\n                data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)\n\n                self.age_mean = data['Age'].mean()\n                self.age_std = data['Age'].std()\n                data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)\n\n                self.weeks_mean = data['Weeks'].mean()\n                self.weeks_std = data['Weeks'].std()\n                data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)\n\n                \n            #Normalization\n            if self.normalization:\n                self.base_week_min = data['Base_week'].min()\n                self.base_week_max = data['Base_week'].max()\n                data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)\n\n                self.base_fvc_min = data['Base_FVC'].min()\n                self.base_fvc_max = data['Base_FVC'].max()\n                data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)\n\n                self.base_percent_min = data['Percent'].min()\n                self.base_percent_max = data['Percent'].max()\n                data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)\n\n                self.age_min = data['Age'].min()\n                self.age_max = data['Age'].max()\n                data['Age'] = normalization(data['Age'],self.age_max,self.age_min)\n\n                self.weeks_min = data['Weeks'].min()\n                self.weeks_max = data['Weeks'].max()\n                data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)\n                \n                self.base_week_min = data['Min_week'].min()\n                self.base_week_max = data['Min_week'].max()\n                data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)\n\n            \n        return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Transform data\ndata_prep = data_preparation(bool_normalization=True)\ntrain = data_prep(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Keep the data_prep object for tabular X_prediction\npickefile = open('data_prep', 'wb')\npickle.dump(data_prep, pickefile)\npickefile.close()\n\ntrain.to_csv('train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Count number of images files for each patient"},{"metadata":{"trusted":true},"cell_type":"code","source":"# list_files = pd.DataFrame(columns=['files'])\n# list_files['Patient'] = [i for i in os.listdir('/kaggle/input/osic-pulmonary-fibrosis-progression/train/')]\n# list_files.loc[:,'files'] = [os.listdir('/kaggle/input/osic-pulmonary-fibrosis-progression/train/'+i) for i in list_files['Patient']]\n# list_files['num_files'] = list_files['files'].apply(lambda x:len(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import pydicom\n\n# def sort_function(x): #Get the files in the right order\n#     return int(x.split('.')[0])\n\n# size_image = pd.DataFrame(columns=['img_names','path', 'patient', 'size'])\n# path = '/kaggle/input/osic-pulmonary-fibrosis-progression/train/'\n# for patient in os.listdir('/kaggle/input/osic-pulmonary-fibrosis-progression/train/'):\n#     list_img_names = sorted([i for i in os.listdir(path+'/'+patient)], key=sort_function)\n#     df = pd.DataFrame({'img_names':list_img_names})\n#     df['path'] = df.apply(lambda x:path+patient+'/'+x.loc['img_names'], axis=1)\n#     df['patient'] = patient\n#     df['size'] = df.apply(lambda x: (pydicom.dcmread(x.loc['path']).data_element('Rows').value, pydicom.dcmread(x.loc['path']).data_element('Rows').value), axis=1)\n#     size_image = size_image.append(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# size_image.to_csv('size_image.csv', index=False)\n# list_files.to_csv('list_files.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}