{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1 align=\"center\"> EDA + Modelling (using tabular data only) </h1>"},{"metadata":{},"cell_type":"markdown","source":"<div align=\"right-justify\">Actually i am making this notebook for beginners. Actually i was receiving some queries how to participate in live competitions. I am using tabular data only and calculating predictions based only upon that. I predicted on test data. You can move forward after using this notebook. I have made this notebook after taking some help from other kagglers notebooks and tried to make this notebook as simple as possible.</div><br> Thanks"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport tempfile\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nfrom IPython.display import display\n\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nsns.set_palette('deep')\nmpl.rcParams['figure.figsize'] = (9,12)\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n\nimport sklearn\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_csv=\"../input/osic-pulmonary-fibrosis-progression/train.csv\"\ndata = {'train_data' : pd.read_csv(train_csv, index_col='Patient',\n                                    dtype={'Patient':'object',\n                                            'Weeks':np.int16,\n                                            'FVC':np.int32,\n                                            'Percent':np.float64,\n                                            'Age':np.int8,\n                                            'Sex':'category',\n                                            'SmokingStatus':'category'})}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data_boolean: One hot encoded columns\n#data_numeric: Numerical data values\ndata['numeric'] = data['train_data'].drop(columns=['Sex', 'SmokingStatus'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# **OneHotEncoding** applayed to column SmokingStatus\n# **Binarizing** column sex to. True if is male, False otherwise\nohe = sklearn.preprocessing.OneHotEncoder(sparse=False, dtype=np.bool)\nsmoker_array = ohe.fit_transform(data['train_data'][['SmokingStatus']])\nsmoker_stack = np.stack(smoker_array, 1)\ndata['boolean'] = pd.DataFrame().assign(isMale = data['train_data'].Sex=='Male',\n                                    currently_smokes = smoker_stack[0],\n                                    ex_smoker = smoker_stack[1],\n                                    never_smoked = smoker_stack[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalization\n# data_normalized: Normalized and encoded data\nscaler = sklearn.preprocessing.RobustScaler()  # To normalize data\ndata['normalized'] = scaler.fit_transform(data['numeric']) # Normalize\ndata['normalized'] = pd.DataFrame(data['normalized'],\n                            data['numeric'].index,\n                            data['numeric'].columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataframe summaries\ndisplay(ohe.categories_)  # smoker categories\ndisplay(data['normalized'].describe())  # numeric variables\ndisplay(data['boolean'].describe())  # As bool \ndisplay(data['boolean'].astype(np.int8).describe())  # booleans as numeric","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Questions to answer:**\n<br>\n\n*Shape of data*<br>\n*is it balanced?<br>\nCorrelation: Weeks-FVC<br>\nPrevalecence: Age - Weeks<br>\nPrevalecence: Age - FVC<br>\nPrevalecence: SmokingStatus - Percent<br>*"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.violinplot(x='SmokingStatus', y='FVC',\n                hue='Sex', split=True,\n                data=data['train_data'],\n                inner=\"quart\", linewidth=1,\n                palette={'Male': \".75\", 'Female': \"pink\"},\n                ci=\"sd\", alpha=.6, height=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.pairplot(data['train_data'], hue='Sex')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Statistics\n# Bootstrap sampling function: multiple_balanced_sampling\ndef multiple_balanced_sampling(data, group_by, epochs=100, samples=1_000):\n    def item_to_front(ls, value):\n        list_ = list(ls)\n        try:\n            val_index = list_.index(value)\n            list_.insert(0, list_.pop(val_index))\n        except ValueError:\n            list_.insert(0, group_by)\n        return list_\n    \n    col_names = item_to_front(data.columns, group_by)\n    def sample_balanced(data, categories = set(data[group_by])):\n        def sampler(data, category, col_names=col_names):\n            sub_df = data[data[group_by]==category]\n            sample = sub_df.sample(samples, replace=True)\n            return sample.groupby([group_by]).mean().reset_index()\n        return np.vstack([sampler(data, cat) for cat in categories])\n    samples_list = np.vstack([sample_balanced(data) for i in range(epochs)])\n    return pd.DataFrame(samples_list, columns=col_names).dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples = 1500      # {type:\"slider\", min:100, max:10000, step:100}\nepochs = 100        # {type:\"slider\", min:10, max:1000, step:10}\ncategory = \"SmokingStatus\" # [\"Sex\", \"SmokingStatus\"] {allow-input: true}\n#\nkey_ = \"bs_\" + category\ndata[key_] = data['numeric'].copy()\ndata[key_] = data['numeric'].join(data['train_data'][category])\ndata[key_] = multiple_balanced_sampling(data[key_], category, samples, epochs)\n# # correcting dtypes\ndata[key_] = data[key_].infer_objects()\ndata[key_][category] = data[key_][category].astype('category')\n# viz\nsns.pairplot(data[key_], hue=category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples = 1300      # {type:\"slider\", min:100, max:10000, step:100}\nepochs = 110        # {type:\"slider\", min:10, max:1000, step:10}\ncategory = \"Sex\"    # [\"Sex\", \"SmokingStatus\"] {allow-input: true}\n#\nkey_ = \"bs_\" + category\ndata[key_] = data['numeric'].copy()\ndata[key_] = data['numeric'].join(data['train_data'][category])\ndata[key_] = multiple_balanced_sampling(data[key_], category, samples, epochs)\n# # correcting dtypes\ndata[key_] = data[key_].infer_objects()\ndata[key_][category] = data[key_][category].astype('category')\n# viz\nsns.pairplot(data[key_], hue=category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalized_data = True # {type:\"boolean\"}\nif normalized_data:\n    data_source = \"normalized\"\nelse:\n    data_source = \"train_data\"\nmethod_1 = \"kendall\" #[\"spearman\", \"kendall\", \"pearson\"]\ncorr_bs_1 = data[data_source].corr(method_1)\ncorr_bs_1 = corr_bs_1.abs()\n\nmethod_2 = \"spearman\" #[\"spearman\", \"kendall\", \"pearson\"]\ncorr_bs_2 = data[data_source].corr(method_2)\ncorr_bs_1 = corr_bs_1.abs()\n\nf, axis = plt.subplots(1, 2, figsize=(10, 4))\naxis[0].set_title(\"Samples balanced by Sex\")\nsns.heatmap(corr_bs_1, cmap='gray', ax = axis[0])\naxis[1].set_title(\"Samples balanced by SmokingStatus\")\nsns.heatmap(corr_bs_2, cmap='gray', ax = axis[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Again loading\ntraindf = pd.read_csv(train_csv)\ntraindf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf.drop(['Percent'], axis=1, inplace=True)\ntraindf.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\n\n# Create a list of unique patients\npatients = list(traindf['Patient'].unique())\nlen(patients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find and append the baseline FVC for each patient\n\nfor i in patients:\n    base_fvc = traindf.loc[(traindf['Patient'] == i) & \n                           (traindf['Weeks'] == min(traindf[traindf['Patient'] == i]['Weeks'], key=abs)),['FVC']]\n    traindf.loc[(traindf['Patient'] == i),'base_fvc'] = int(np.asarray(base_fvc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfeatures = traindf[['Patient', 'Sex', 'SmokingStatus']]\nfeatures = pd.get_dummies(features, columns=['Sex', 'SmokingStatus'])\n\nfeatures['age'] = (traindf['Age'] - traindf['Age'].min()) / \\\n                      (traindf['Age'].max() - traindf['Age'].min())\n\nfeatures['weeks'] = (traindf['Weeks'] - traindf['Weeks'].min()) / \\\n                      (traindf['Weeks'].max() - traindf['Age'].min())\n\nfeatures['base_fvc'] = (traindf['base_fvc'] - traindf['base_fvc'].min()) / \\\n                      (traindf['base_fvc'].max() - traindf['base_fvc'].min())\n    \nfeatures['fvc'] = traindf['FVC']\nfeatures","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features.to_pickle('tabular_features.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import train_test_split\ntrain_inds, test_inds = next(GroupShuffleSplit(test_size=.20, n_splits=2, random_state = 42).split(features, groups=features['Patient']))\ntrain = features.iloc[train_inds]\ntest = features.iloc[test_inds]\ntrain.drop('Patient', axis=1, inplace=True)\ntest.drop('Patient', axis=1, inplace=True)\nX_train = train.loc[:,:'weeks']\ny_train = train['fvc']\nX_test = test.loc[:,:'weeks']\ny_test = test['fvc']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = features[['Sex_Female', 'Sex_Male', 'SmokingStatus_Currently smokes', 'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked', 'age', 'weeks']]\ny = features['fvc']\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\ny_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build the model\nimport tensorflow as tf\nfrom tensorflow.keras import datasets, layers, models\nmodel = models.Sequential()\n\nmodel.add(layers.Dense(64, activation='relu', input_shape=(7,)))\nmodel.add(layers.Dense(32, activation='relu'))\n#model.add(layers.Dropout(0.3))\n#model.add(layers.Dense(16, activation='relu', kernel_regularizer=tf.keras.regularizers.l1(0)))\nmodel.add(layers.Dense(1, activation='linear'))\n\n# kernel_regularizer=tf.keras.regularizers.l2(0.001)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n             loss='mape',\n             metrics=['mae', 'mse', 'mape'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Specify callbacks for tensorboard\nfrom datetime import datetime\nlog_dir = \"logs/fit/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\ntensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n\n# Specify early stopping callback\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, mode='auto', restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=300,\n                    validation_data=(X_test, y_test),\n                    callbacks=[tensorboard_callback],\n                    verbose=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(X_test, batch_size=1)\npredictions\na = plt.axes(aspect='equal')\nplt.scatter(predictions, y_test, edgecolors=(0, 0, 0))\nplt.xlabel('True Values')\nplt.ylabel('Predictions')\nlims = [1500, 4000]\nplt.xlim(lims)\nplt.ylim(lims)\n_ = plt.plot(lims, lims)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}