{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading packages","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold, RandomizedSearchCV\nfrom sklearn.metrics import make_scorer, roc_auc_score\nfrom sklearn.ensemble import RandomForestClassifier\nimport seaborn as sns\nfrom sklearn.impute import SimpleImputer\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-15T04:50:39.075652Z","iopub.execute_input":"2021-09-15T04:50:39.076056Z","iopub.status.idle":"2021-09-15T04:50:39.083323Z","shell.execute_reply.started":"2021-09-15T04:50:39.076022Z","shell.execute_reply":"2021-09-15T04:50:39.082066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading train and test sets","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/glcm-training/glcm_training.csv')\ntest = pd.read_csv('../input/siimglcmtest/glcm_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.085666Z","iopub.execute_input":"2021-09-15T04:50:39.086194Z","iopub.status.idle":"2021-09-15T04:50:39.30399Z","shell.execute_reply.started":"2021-09-15T04:50:39.086153Z","shell.execute_reply":"2021-09-15T04:50:39.302937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Previewing training and testing sets","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.305117Z","iopub.execute_input":"2021-09-15T04:50:39.305419Z","iopub.status.idle":"2021-09-15T04:50:39.330975Z","shell.execute_reply.started":"2021-09-15T04:50:39.305389Z","shell.execute_reply":"2021-09-15T04:50:39.329895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.33249Z","iopub.execute_input":"2021-09-15T04:50:39.332802Z","iopub.status.idle":"2021-09-15T04:50:39.356675Z","shell.execute_reply.started":"2021-09-15T04:50:39.332763Z","shell.execute_reply":"2021-09-15T04:50:39.355439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Determining distribution of missing values in train set","metadata":{}},{"cell_type":"code","source":"percent_missing = train.isnull().sum() * 100 / len(train)\nmissing_vals_df = pd.DataFrame({'Percent Missing': percent_missing})\nprint(missing_vals_df)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.360426Z","iopub.execute_input":"2021-09-15T04:50:39.360761Z","iopub.status.idle":"2021-09-15T04:50:39.380614Z","shell.execute_reply.started":"2021-09-15T04:50:39.360728Z","shell.execute_reply":"2021-09-15T04:50:39.379343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"Codifying categorical variables and normalizing numeric features","metadata":{}},{"cell_type":"markdown","source":"### Sectioning into categorical and numerical features","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.38258Z","iopub.execute_input":"2021-09-15T04:50:39.382911Z","iopub.status.idle":"2021-09-15T04:50:39.40373Z","shell.execute_reply.started":"2021-09-15T04:50:39.382878Z","shell.execute_reply":"2021-09-15T04:50:39.402805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_features = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','target']\nprint(\"Numerical features:\", num_features)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.4051Z","iopub.execute_input":"2021-09-15T04:50:39.40554Z","iopub.status.idle":"2021-09-15T04:50:39.412063Z","shell.execute_reply.started":"2021-09-15T04:50:39.405499Z","shell.execute_reply":"2021-09-15T04:50:39.41085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining preprocessor","metadata":{}},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import MinMaxScaler\n\nclass PreprocessTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, num_features):\n\n        self.num_features = num_features\n    \n    def fit(self, X, y=None):\n        return self\n  \n    def transform(self, X, y=None): \n        dataframe = X.copy()\n\n        # Normalize numerical features\n        scaler = MinMaxScaler()\n        dataframe[self.num_features] = scaler.fit_transform(dataframe[num_features])\n        \n        return dataframe","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.413728Z","iopub.execute_input":"2021-09-15T04:50:39.414162Z","iopub.status.idle":"2021-09-15T04:50:39.425814Z","shell.execute_reply.started":"2021-09-15T04:50:39.41412Z","shell.execute_reply":"2021-09-15T04:50:39.424515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing train set","metadata":{}},{"cell_type":"code","source":"# Preprocessing categorical and numerical features\ntrain_processed = PreprocessTransformer(num_features).transform(X = train)\n\n# Imputing missing values \n#train_noNan = pd.DataFrame(SimpleImputer().fit_transform(train_processed))\n#train_noNan.columns = train_processed.columns\n#\n#train_noNan.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.427332Z","iopub.execute_input":"2021-09-15T04:50:39.42776Z","iopub.status.idle":"2021-09-15T04:50:39.691608Z","shell.execute_reply.started":"2021-09-15T04:50:39.427715Z","shell.execute_reply":"2021-09-15T04:50:39.690541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#percent_missing = train_noNan.isnull().sum() * 100 / len(train_noNan)\n#missing_vals_df = pd.DataFrame({'Percent Missing': percent_missing})\n#print(missing_vals_df)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.693143Z","iopub.execute_input":"2021-09-15T04:50:39.693463Z","iopub.status.idle":"2021-09-15T04:50:39.697664Z","shell.execute_reply.started":"2021-09-15T04:50:39.693431Z","shell.execute_reply":"2021-09-15T04:50:39.696412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest Classifier","metadata":{}},{"cell_type":"markdown","source":"## Splitting into X (input) and y (output)","metadata":{}},{"cell_type":"code","source":"X_train = train_processed.copy().drop(columns = ['target','image_name'])\ny_train = train_processed.copy().drop(columns = ['image_name'])\ny_train = train_processed.copy()['target']\n\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.699454Z","iopub.execute_input":"2021-09-15T04:50:39.699897Z","iopub.status.idle":"2021-09-15T04:50:39.749041Z","shell.execute_reply.started":"2021-09-15T04:50:39.699853Z","shell.execute_reply":"2021-09-15T04:50:39.747651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter adjustment","metadata":{}},{"cell_type":"code","source":"search_space = [\n  {\n     'max_depth': [10, 20, 30, 40, 50, 60, None],\n     'max_features': ['auto', 'sqrt'],\n     'min_samples_leaf': [1, 2, 4],\n     'min_samples_split': [2, 5, 10],\n     'n_estimators': [200, 400, 600, 800, 1000]\n  }\n]\n\ncv_method = StratifiedKFold(n_splits=4, shuffle = True, random_state=0)\nscoring = {'AUC':make_scorer(roc_auc_score)}","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.750622Z","iopub.execute_input":"2021-09-15T04:50:39.751059Z","iopub.status.idle":"2021-09-15T04:50:39.758899Z","shell.execute_reply.started":"2021-09-15T04:50:39.750993Z","shell.execute_reply":"2021-09-15T04:50:39.757749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fitting random forest classifier to train set","metadata":{}},{"cell_type":"code","source":"optimizer = RandomizedSearchCV(\n  estimator = RandomForestClassifier(),\n  param_distributions=search_space,\n  cv=cv_method,\n  scoring=scoring,\n  refit='AUC',\n  return_train_score = True,\n  verbose=1,\n  n_iter = 50,\n)\n\n# Approximately 1 hour run time with GPU assistance\nrf_model = optimizer.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T04:50:39.760429Z","iopub.execute_input":"2021-09-15T04:50:39.760772Z","iopub.status.idle":"2021-09-15T08:06:12.011492Z","shell.execute_reply.started":"2021-09-15T04:50:39.76074Z","shell.execute_reply":"2021-09-15T08:06:12.0098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display mean AUC score\noptimizer.cv_results_['mean_test_AUC'].mean()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:12.013211Z","iopub.execute_input":"2021-09-15T08:06:12.013589Z","iopub.status.idle":"2021-09-15T08:06:12.023423Z","shell.execute_reply.started":"2021-09-15T08:06:12.013554Z","shell.execute_reply":"2021-09-15T08:06:12.022296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display most important parameters\noptimizer.best_params_","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:12.025305Z","iopub.execute_input":"2021-09-15T08:06:12.025979Z","iopub.status.idle":"2021-09-15T08:06:12.044102Z","shell.execute_reply.started":"2021-09-15T08:06:12.025926Z","shell.execute_reply":"2021-09-15T08:06:12.042749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Displaying feature importance","metadata":{}},{"cell_type":"code","source":"features = X_train.columns\nimp_dict = {features[i]:optimizer.best_estimator_.feature_importances_[i] for i in range(len(features))}\nimp_dict = sorted(imp_dict.items(), key=lambda x: x[1])\nprint(imp_dict)\n\nplt.bar(*zip(*imp_dict))\nplt.xticks(rotation=\"vertical\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:12.04535Z","iopub.execute_input":"2021-09-15T08:06:12.045655Z","iopub.status.idle":"2021-09-15T08:06:13.313563Z","shell.execute_reply.started":"2021-09-15T08:06:12.045626Z","shell.execute_reply":"2021-09-15T08:06:13.3124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting on test set","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:13.314895Z","iopub.execute_input":"2021-09-15T08:06:13.315211Z","iopub.status.idle":"2021-09-15T08:06:13.341109Z","shell.execute_reply.started":"2021-09-15T08:06:13.315181Z","shell.execute_reply":"2021-09-15T08:06:13.339601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_copy = test.copy().drop(columns = ['image_name'])\n\n\nnum_features = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o']\ntest_processed = PreprocessTransformer(num_features).transform(X = test_copy)\n\ntest_noNan = pd.DataFrame(SimpleImputer().fit_transform(test_processed))\ntest_noNan.columns = test_processed.columns\n\ntest_processed.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:13.342334Z","iopub.execute_input":"2021-09-15T08:06:13.342703Z","iopub.status.idle":"2021-09-15T08:06:13.4232Z","shell.execute_reply.started":"2021-09-15T08:06:13.342668Z","shell.execute_reply":"2021-09-15T08:06:13.421989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = rf_model.predict_proba(test_processed[features])\npd.DataFrame(y_pred).head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:13.424786Z","iopub.execute_input":"2021-09-15T08:06:13.425139Z","iopub.status.idle":"2021-09-15T08:06:13.805433Z","shell.execute_reply.started":"2021-09-15T08:06:13.425103Z","shell.execute_reply":"2021-09-15T08:06:13.804418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_malignant = [p[1] for p in y_pred]","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:13.806703Z","iopub.execute_input":"2021-09-15T08:06:13.807024Z","iopub.status.idle":"2021-09-15T08:06:13.820051Z","shell.execute_reply.started":"2021-09-15T08:06:13.80698Z","shell.execute_reply":"2021-09-15T08:06:13.818945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['image_name'] = test.image_name.values\nsubmission['target'] = y_pred_malignant\n\nsubmission.to_csv('submission.csv',index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T08:06:13.821546Z","iopub.execute_input":"2021-09-15T08:06:13.821867Z","iopub.status.idle":"2021-09-15T08:06:13.902942Z","shell.execute_reply.started":"2021-09-15T08:06:13.821838Z","shell.execute_reply":"2021-09-15T08:06:13.901922Z"},"trusted":true},"execution_count":null,"outputs":[]}]}