{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import the necessary python packages","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, GridSearchCV\nfrom sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import StandardScaler\n    \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\n\n%matplotlib inline\n\n\nprint(os.listdir(\"../input\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T15:17:04.362755Z","iopub.execute_input":"2022-08-10T15:17:04.364010Z","iopub.status.idle":"2022-08-10T15:17:04.380731Z","shell.execute_reply.started":"2022-08-10T15:17:04.363424Z","shell.execute_reply":"2022-08-10T15:17:04.379313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import and modify the data for training the model","metadata":{}},{"cell_type":"markdown","source":"## Import the data","metadata":{}},{"cell_type":"code","source":"dataForModel = pd.read_csv(\"../input/train.csv\")","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-08-10T15:17:04.382641Z","iopub.execute_input":"2022-08-10T15:17:04.383178Z","iopub.status.idle":"2022-08-10T15:17:04.407558Z","shell.execute_reply.started":"2022-08-10T15:17:04.383124Z","shell.execute_reply":"2022-08-10T15:17:04.406503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Shuffle the data","metadata":{}},{"cell_type":"markdown","source":"#### This is to ensure that when you will split the data for training the model into a training and validation set, both sets will contain the whole breadth of data.","metadata":{}},{"cell_type":"code","source":"# Shuffle data\ndataForModel = shuffle(dataForModel, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.409552Z","iopub.execute_input":"2022-08-10T15:17:04.410334Z","iopub.status.idle":"2022-08-10T15:17:04.419309Z","shell.execute_reply.started":"2022-08-10T15:17:04.410252Z","shell.execute_reply":"2022-08-10T15:17:04.418358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the data and its main statistics","metadata":{}},{"cell_type":"code","source":"# Take a look at the data\ndataForModel","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.422431Z","iopub.execute_input":"2022-08-10T15:17:04.423277Z","iopub.status.idle":"2022-08-10T15:17:04.498985Z","shell.execute_reply.started":"2022-08-10T15:17:04.422983Z","shell.execute_reply":"2022-08-10T15:17:04.497775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Count the number of NaN entries","metadata":{}},{"cell_type":"code","source":"dataForModel.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.502294Z","iopub.execute_input":"2022-08-10T15:17:04.502664Z","iopub.status.idle":"2022-08-10T15:17:04.512054Z","shell.execute_reply.started":"2022-08-10T15:17:04.502596Z","shell.execute_reply":"2022-08-10T15:17:04.511065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the statistics of the data","metadata":{}},{"cell_type":"code","source":"dataForModel.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.514197Z","iopub.execute_input":"2022-08-10T15:17:04.514635Z","iopub.status.idle":"2022-08-10T15:17:04.567082Z","shell.execute_reply.started":"2022-08-10T15:17:04.514587Z","shell.execute_reply":"2022-08-10T15:17:04.565900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop unnecessary columns","metadata":{}},{"cell_type":"code","source":"# drop unnecessary columns: PassengerId\ndataForModel.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.568844Z","iopub.execute_input":"2022-08-10T15:17:04.569472Z","iopub.status.idle":"2022-08-10T15:17:04.686291Z","shell.execute_reply.started":"2022-08-10T15:17:04.569406Z","shell.execute_reply":"2022-08-10T15:17:04.685367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Deal with missing/NaN values","metadata":{}},{"cell_type":"markdown","source":"#### Dealing with missing values - Age: When dealing with missing/NaN values for numerical variables, you could either remove them or substitute them with the average or median. For the Age, removal is not a viable option. The NaN for this variable account for almost 20 % of all entries. Substitution is a better option. I choose the Age median value.","metadata":{}},{"cell_type":"code","source":"dataForModel['Age'].fillna(dataForModel['Age'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.688244Z","iopub.execute_input":"2022-08-10T15:17:04.688961Z","iopub.status.idle":"2022-08-10T15:17:04.697042Z","shell.execute_reply.started":"2022-08-10T15:17:04.688886Z","shell.execute_reply":"2022-08-10T15:17:04.695649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dealing with missing values - Embarked: When dealing with missing/NaN values for categorical variables, you could either remove them or substitute them with another value. For the Embarked variable, removal is the best option as there are only two NaN entries.","metadata":{}},{"cell_type":"code","source":"dataForModel.dropna(how = \"any\", inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.699283Z","iopub.execute_input":"2022-08-10T15:17:04.699782Z","iopub.status.idle":"2022-08-10T15:17:04.713973Z","shell.execute_reply.started":"2022-08-10T15:17:04.699587Z","shell.execute_reply":"2022-08-10T15:17:04.712601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Re-count the number of NaN entries","metadata":{}},{"cell_type":"code","source":"dataForModel.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.715933Z","iopub.execute_input":"2022-08-10T15:17:04.716513Z","iopub.status.idle":"2022-08-10T15:17:04.728170Z","shell.execute_reply.started":"2022-08-10T15:17:04.716429Z","shell.execute_reply":"2022-08-10T15:17:04.726913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Deal with categorical variables using One Hot Encoding","metadata":{}},{"cell_type":"code","source":"categorical_vars = [\"Sex\", \"Embarked\"]\noneHotEncoder = OneHotEncoder(sparse=False, drop=\"first\")\n\ndataForModelEncoded = oneHotEncoder.fit_transform(dataForModel[categorical_vars])\nencoderFeatureNames = oneHotEncoder.get_feature_names(categorical_vars)\n\ndataForModelEncoded = pd.DataFrame(dataForModelEncoded,columns = encoderFeatureNames)\ndataForModel = pd.concat([dataForModel.reset_index(drop=True),dataForModelEncoded.reset_index(drop=True)],axis=1)\ndataForModel.drop(categorical_vars,axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.730112Z","iopub.execute_input":"2022-08-10T15:17:04.730688Z","iopub.status.idle":"2022-08-10T15:17:04.749568Z","shell.execute_reply.started":"2022-08-10T15:17:04.730622Z","shell.execute_reply":"2022-08-10T15:17:04.748566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataForModel","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.751582Z","iopub.execute_input":"2022-08-10T15:17:04.752322Z","iopub.status.idle":"2022-08-10T15:17:04.814049Z","shell.execute_reply.started":"2022-08-10T15:17:04.752250Z","shell.execute_reply":"2022-08-10T15:17:04.812784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate the class balance","metadata":{}},{"cell_type":"markdown","source":"#### It is important to evaluate whether the data used for training the model, contains a balanced number of entries corresponding to the classes we ought to predict.","metadata":{}},{"cell_type":"code","source":"dataForModel[\"Survived\"].value_counts(normalize = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.815657Z","iopub.execute_input":"2022-08-10T15:17:04.816010Z","iopub.status.idle":"2022-08-10T15:17:04.826701Z","shell.execute_reply.started":"2022-08-10T15:17:04.815945Z","shell.execute_reply":"2022-08-10T15:17:04.825817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The data model is not very balanced. Therefore a better estimation of the model accuracy comes from the F1_score rather than the accuracy_score.","metadata":{}},{"cell_type":"markdown","source":"## Split the available data into a training and validation set","metadata":{}},{"cell_type":"code","source":"X = dataForModel.drop([\"Survived\"], axis = 1)\ny = dataForModel[\"Survived\"]\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42, stratify = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.828119Z","iopub.execute_input":"2022-08-10T15:17:04.828587Z","iopub.status.idle":"2022-08-10T15:17:04.846769Z","shell.execute_reply.started":"2022-08-10T15:17:04.828534Z","shell.execute_reply":"2022-08-10T15:17:04.845810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.850879Z","iopub.execute_input":"2022-08-10T15:17:04.851583Z","iopub.status.idle":"2022-08-10T15:17:04.913244Z","shell.execute_reply.started":"2022-08-10T15:17:04.851510Z","shell.execute_reply":"2022-08-10T15:17:04.912087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:17:04.915303Z","iopub.execute_input":"2022-08-10T15:17:04.915989Z","iopub.status.idle":"2022-08-10T15:17:04.973457Z","shell.execute_reply.started":"2022-08-10T15:17:04.915896Z","shell.execute_reply":"2022-08-10T15:17:04.972111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optimize parameters with a grid search","metadata":{}},{"cell_type":"markdown","source":"#### Choose parameter:\n#### --) OneD = True --> One-dimensional grid search\n#### --) OneD = False --> Multi-dimensional grid search","metadata":{}},{"cell_type":"code","source":"oneD = True","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:12.863458Z","iopub.execute_input":"2022-08-10T15:43:12.864100Z","iopub.status.idle":"2022-08-10T15:43:12.870815Z","shell.execute_reply.started":"2022-08-10T15:43:12.863809Z","shell.execute_reply":"2022-08-10T15:43:12.869418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (oneD):\n    max_depth_list = list(range(1,15))\n    accuracy_scores = []\n    for depth in max_depth_list:\n        clf = DecisionTreeClassifier(max_depth = depth, random_state = 42)\n        clf.fit(X_train, y_train)\n        y_pred = clf. predict(X_test)\n        accuracy = f1_score(y_test, y_pred)\n        accuracy_scores.append(accuracy)\n        \n    max_accuracy = max(accuracy_scores)\n    max_accuracy_idx = accuracy_scores.index(max_accuracy)\n    optimal_depth = max_depth_list[max_accuracy_idx]\n    \n    # Plot of max depths\n    plt.plot(max_depth_list, accuracy_scores)\n    plt.scatter(optimal_depth, max_accuracy, marker = \"x\", color = \"red\")\n    plt.title(f\"Accuracy (F1 Score) by Max Depth \\n Optimal Tree Depth: {optimal_depth} (Accuracy: {round(max_accuracy)})\")\n    plt.xlabel(\"Max Depth of Decision Tree\")\n    plt.ylabel(\"Accuracy (F1 Score)\")\n    plt.tight_layout()\n    plt.show()\n    \n    classifier = DecisionTreeClassifier(random_state = 42, max_depth = 5)\nelse:\n    parameter_grid = {\n        'max_depth' : list(range(1,6)),\n        'min_samples_split': list(range(2,7)),\n        'min_samples_leaf': list(range(2,7)),\n        'max_features': ['sqrt', 'auto', 'log2']\n    }\n    \n    clf = DecisionTreeClassifier()\n    cross_validation = StratifiedKFold(n_splits=5)\n    grid_search = GridSearchCV(clf, scoring='f1', param_grid=parameter_grid, verbose=1)\n    grid_search.fit(X_train, y_train)\n    model = grid_search\n    parameters = grid_search.best_params_\n    print('Best score: {}'.format(grid_search.best_score_))\n    print('Best parameters: {}'.format(grid_search.best_params_))\n    \n    classifier = DecisionTreeClassifier(random_state=42, **parameters)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:15.021406Z","iopub.execute_input":"2022-08-10T15:43:15.021783Z","iopub.status.idle":"2022-08-10T15:43:15.588139Z","shell.execute_reply.started":"2022-08-10T15:43:15.021736Z","shell.execute_reply":"2022-08-10T15:43:15.586786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"classifier.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:18.821563Z","iopub.execute_input":"2022-08-10T15:43:18.822401Z","iopub.status.idle":"2022-08-10T15:43:18.837961Z","shell.execute_reply.started":"2022-08-10T15:43:18.822329Z","shell.execute_reply":"2022-08-10T15:43:18.836642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate the model accuracy","metadata":{}},{"cell_type":"markdown","source":"#### Confusion matrix","metadata":{}},{"cell_type":"code","source":"y_pred = classifier.predict(X_test)\nconf_matrix = confusion_matrix(y_test, y_pred)\nprint(conf_matrix)\n\nplt.style.use(\"seaborn-poster\")\nplt.matshow(conf_matrix, cmap = \"coolwarm\")\nplt.gca().xaxis.tick_bottom()\nplt.title(\"Confusion Matrix\")\nplt.ylabel(\"Actual Class\")\nplt.xlabel(\"Predicted Class\")\nfor (i, j), corr_value in np.ndenumerate(conf_matrix):\n    plt.text(j, i, corr_value, ha = \"center\", va = \"center\", fontsize = 20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:20.861002Z","iopub.execute_input":"2022-08-10T15:43:20.861667Z","iopub.status.idle":"2022-08-10T15:43:21.165149Z","shell.execute_reply.started":"2022-08-10T15:43:20.861600Z","shell.execute_reply":"2022-08-10T15:43:21.163514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Accuracy Score, Precision, Recall and F1-score","metadata":{}},{"cell_type":"code","source":"accuracyScore = accuracy_score(y_test, y_pred)\nPrecisionScore = precision_score(y_test, y_pred)\nRecallScore = recall_score(y_test, y_pred)\nF1Score = f1_score(y_test, y_pred)\nprint(accuracyScore, PrecisionScore, RecallScore, F1Score)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:35:58.829779Z","iopub.execute_input":"2022-08-10T15:35:58.830254Z","iopub.status.idle":"2022-08-10T15:35:58.842228Z","shell.execute_reply.started":"2022-08-10T15:35:58.830180Z","shell.execute_reply":"2022-08-10T15:35:58.841248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import the test data","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\")\ntest.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:36:01.707928Z","iopub.execute_input":"2022-08-10T15:36:01.708455Z","iopub.status.idle":"2022-08-10T15:36:01.739375Z","shell.execute_reply.started":"2022-08-10T15:36:01.708374Z","shell.execute_reply":"2022-08-10T15:36:01.738135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply the same data preparation processing to the test data","metadata":{}},{"cell_type":"code","source":"# Apply the data preparation also to the test set\ntest['Age'].fillna(test['Age'].median(), inplace=True)\ntest['Fare'].fillna(test['Fare'].median(), inplace=True)\nindex = test['PassengerId']\ntest.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis = 1, inplace = True)\n\n# Deal with categorical variables\ncategorical_vars = [\"Sex\", \"Embarked\"]\noneHotEncoder = OneHotEncoder(sparse=False, drop=\"first\")\n\ntestEncoded = oneHotEncoder.fit_transform(test[categorical_vars])\nencoderFeatureNames = oneHotEncoder.get_feature_names(categorical_vars)\n\ntestEncoded = pd.DataFrame(testEncoded,columns = encoderFeatureNames)\ntest = pd.concat([test.reset_index(drop=True),testEncoded.reset_index(drop=True)],axis=1)\ntest.drop(categorical_vars,axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:36:13.988643Z","iopub.execute_input":"2022-08-10T15:36:13.989312Z","iopub.status.idle":"2022-08-10T15:36:14.015178Z","shell.execute_reply.started":"2022-08-10T15:36:13.989227Z","shell.execute_reply":"2022-08-10T15:36:14.013875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate your predictions","metadata":{}},{"cell_type":"code","source":"pred = pd.DataFrame({\"PassengerId\": index, \"Survived\": classifier.predict(test)})\npred.reset_index(drop=True, inplace = True)\npred.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:37:24.586709Z","iopub.execute_input":"2022-08-10T15:37:24.587351Z","iopub.status.idle":"2022-08-10T15:37:24.597521Z","shell.execute_reply.started":"2022-08-10T15:37:24.587279Z","shell.execute_reply":"2022-08-10T15:37:24.596375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(\"/kaggle/working/\"))\n#os.remove(\"/kaggle/working/submission.csv\")\n#print(os.listdir(\"/kaggle/working/\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:37:28.667077Z","iopub.execute_input":"2022-08-10T15:37:28.667741Z","iopub.status.idle":"2022-08-10T15:37:28.673460Z","shell.execute_reply.started":"2022-08-10T15:37:28.667686Z","shell.execute_reply":"2022-08-10T15:37:28.672399Z"},"trusted":true},"execution_count":null,"outputs":[]}]}