{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport category_encoders as ce \nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.linear_model import Ridge, Lasso, RidgeClassifier\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nimport lightgbm as lgb\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn import metrics\nfrom sklearn.feature_selection import SelectFromModel\nfrom collections import Counter\nimport pickle\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.pipeline import FeatureUnion\nfrom sklearn.impute import MissingIndicator\nfrom sklearn.model_selection import KFold\nfrom sklearn.feature_selection import RFECV\nfrom sklearn.inspection import permutation_importance\nfrom category_encoders import TargetEncoder, LeaveOneOutEncoder\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom functools import reduce\nfrom sklearn.linear_model import LogisticRegression\nimport optuna\nfrom optuna.samplers import RandomSampler, GridSampler, TPESampler\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.neural_network import MLPClassifier","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASEPATH = \"../input/siim-isic-melanoma-classification\"\nos.chdir(BASEPATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dftrain = pd.read_csv('train.csv') \ndftest = pd.read_csv('test.csv') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dftrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dftest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create column transfomer \ndef get_ct(cat_features, num_features):\n    numeric_transformer1 = FeatureUnion(transformer_list=[\n            ('imputer',  SimpleImputer(strategy='constant',fill_value=-1)),\n            ('indicator',MissingIndicator())\n            ])\n    ce_cat = ce.OneHotEncoder(cols=cat_features,handle_unknown='value',handle_missing='value')\n    ct1 = ColumnTransformer(\n            transformers=[\n                ('cat',ce_cat,cat_features),\n                ('num',numeric_transformer1,num_features),\n                ],remainder = 'drop')\n    clf = Pipeline(steps=[('preprocessor', ct1)])\n    return clf ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = dftrain[['sex','age_approx','anatom_site_general_challenge']]\ny_train = dftrain['target']\nimage_names = dftrain[['image_name']]\nX_test = dftest[['sex','age_approx','anatom_site_general_challenge']]\ncat_features = ['sex','anatom_site_general_challenge']\nnum_features = ['age_approx']\nct_trans = get_ct(cat_features,num_features)\nX_train = ct_trans.fit_transform(X_train)\nX_test = ct_trans.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name1 = 'Ridge'\nstudy_ridge = optuna.create_study(study_name=study_name1,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Train Ridge**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_ridge(trial):    \n\n    C = trial.suggest_loguniform('alpha',1e-7,10)\n    b = int(trial.suggest_loguniform('b',1,32))\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    model = RidgeClassifier(alpha=C, class_weight={0:1,1:b},random_state=0)\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        model, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_ridge.optimize(opt_ridge, n_trials=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_ridge.trials))\ntrial_ridge = study_ridge.best_trial\nprint('Best score : {}'.format(-trial_ridge.value))\nfor key, value in trial_ridge.params.items():\n    print(\"    {}: {}\".format(key, value))\nalpha_RID = list(trial_ridge.params.items())[0][1]\nb = int(list(trial_ridge.params.items())[1][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Train LightGBM**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_binary_lgb(label, pred):\n    def robust_pow(num_base, num_pow):\n        # numpy does not permit negative numbers to fractional power\n        # use this to perform the power algorithmic\n        return np.sign(num_base) * (np.abs(num_base)) ** (num_pow)\n    \n    gamma_indct = 2.0\n    # retrieve data from dtrain matrix\n    #label = dtrain.label\n    # compute the prediction with sigmoid\n    sigmoid_pred = 1.0 / (1.0 + np.exp(-pred))\n    # gradient\n    # complex gradient with different parts\n    g1 = sigmoid_pred * (1 - sigmoid_pred)\n    g2 = label + ((-1) ** label) * sigmoid_pred\n    g3 = sigmoid_pred + label - 1\n    g4 = 1 - label - ((-1) ** label) * sigmoid_pred\n    g5 = label + ((-1) ** label) * sigmoid_pred\n    # combine the gradient\n    grad = gamma_indct * g3 * robust_pow(g2, gamma_indct) * np.log(g4 + 1e-9) + \\\n           ((-1) ** label) * robust_pow(g5, (gamma_indct + 1))\n    # combine the gradient parts to get hessian components\n    hess_1 = robust_pow(g2, gamma_indct) + \\\n             gamma_indct * ((-1) ** label) * g3 * robust_pow(g2, (gamma_indct - 1))\n    hess_2 = ((-1) ** label) * g3 * robust_pow(g2, gamma_indct) / g4\n    # get the final 2nd order derivative\n    hess = ((hess_1 * np.log(g4 + 1e-9) - hess_2) * gamma_indct +\n            (gamma_indct + 1) * robust_pow(g5, gamma_indct)) * g1\n\n    return grad, hess","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name2 = 'lgb'\nstudy_lgb = optuna.create_study(study_name=study_name2,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_lgb(trial):    \n\n    num_leaves = int(trial.suggest_loguniform(\"num_leaves\", 3,20))\n    subsample =  trial.suggest_discrete_uniform('bfrac',0.6,1.0,q=0.1),\n    subsample_freq = int(trial.suggest_discrete_uniform('bfreq',1,5,q=1.0)),\n    colsample_bytree = trial.suggest_discrete_uniform('feature',0.8,1.0,q=0.05),\n    reg_lambda = trial.suggest_loguniform(\"lambda_l2\", 1, 10)\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    clf  = lgb.LGBMClassifier(objective=focal_binary_lgb,random_state=0,\n                                                        num_leaves = num_leaves,\n                                                         subsample=subsample,subsample_freq=subsample_freq,\n                                                        colsample_bytree=colsample_bytree,reg_lambda=reg_lambda)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        clf, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_lgb.optimize(opt_lgb, n_trials=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_lgb.trials))\ntrial_lgb = study_lgb.best_trial\nprint('Best score : {}'.format(-trial_lgb.value))\nfor key, value in trial_lgb.params.items():\n    print(\"    {}: {}\".format(key, value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_leaves = int(list(trial_lgb.params.items())[0][1])\nbfrac = list(trial_lgb.params.items())[1][1]\nbfreq = int(list(trial_lgb.params.items())[2][1])\nfeature =  list(trial_lgb.params.items())[3][1]\nlambda_l2 = list(trial_lgb.params.items())[4][1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Train NN**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name3 = 'nn'\nstudy_nn = optuna.create_study(study_name=study_name3,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_nn(trial):    \n\n    alpha = trial.suggest_loguniform('alpha',1e-6,10)\n    z = int(trial.suggest_loguniform('z',4,32))\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    model= MLPClassifier(hidden_layer_sizes=[z],alpha=alpha,random_state=0,tol=1e-4,max_iter=200)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        model, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_nn.optimize(opt_nn, n_trials=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_nn.trials))\ntrial_nn = study_nn.best_trial\nprint('Best score : {}'.format(-trial_nn.value))\nfor key, value in trial_nn.params.items():\n    print(\"    {}: {}\".format(key, value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"alpha_nn = list(trial_nn.params.items())[0][1]\nz = int(list(trial_nn.params.items())[1][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Predictions #","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finalize all models \nmodel_final_Ridge = RidgeClassifier(alpha=alpha_RID,class_weight={0:1,1:b},random_state=0)\nmodel_final_LGB = lgb.LGBMClassifier(objective=focal_binary_lgb,random_state=0,\n                                                        num_leaves = num_leaves,\n                                                         subsample=bfrac,subsample_freq=bfreq,\n                                                        colsample_bytree=feature,reg_lambda=lambda_l2)\nmodel_final_NN = MLPClassifier(hidden_layer_sizes=[z],alpha=alpha_nn,random_state=0,tol=1e-4,max_iter=200)\n\n# Do probability calibration for all models\n# https://scikit-learn.org/stable/modules/calibration.html\nkFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\nclf_sigmoid_Ridge = CalibratedClassifierCV(model_final_Ridge, cv=kFold, method=\"sigmoid\")\nclf_sigmoid_LGB = CalibratedClassifierCV(model_final_LGB, cv=kFold, method=\"sigmoid\")\nclf_sigmoid_NN = CalibratedClassifierCV(model_final_NN, cv=kFold, method=\"sigmoid\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_sigmoid_Ridge.fit(X_train, y_train)\nclf_sigmoid_LGB.fit(X_train, y_train)\nclf_sigmoid_NN.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make Predictions\ny_pred_Ridge = clf_sigmoid_Ridge.predict_proba(X_test)[:,1]\ny_pred_LGB = clf_sigmoid_LGB.predict_proba(X_test)[:,1]\ny_pred_NN = clf_sigmoid_NN.predict_proba(X_test)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('sample_submission.csv')\nsubmission['target'] = (y_pred_Ridge + y_pred_LGB + y_pred_NN) / 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}