{"cells":[{"metadata":{"_uuid":"79700cc0f69c9f6ea59e1466a679af30e7d3a677"},"cell_type":"markdown","source":"![](https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRkYM0u6uIEJ166J76eddNETmwzTBluf2yMdLWFvKn4aABEwb8z)\n\n---\nOutline of Notebook\n\n---\n* [1.Loading Important Library](#1.Loading-Important-Library)\n* [2.Helping Function](#2.Helping-Function)\n* [3.Feature Extraction](#3.Feature-Extraction)\n    1. [1.Feature Extract from training set file](#1.Feature-Extract-from-training-set-file)\n    2. [2.Feature Extract from training meta file](#2.Feature-Extract-from-training-meta-file)\n* [4.Parameter Tuning](#4.Parameter-Tuning)\n* [5.Model Design](#5.Model-Design)\n* [6.Model Tuning](#6.Model-Tuning)\n* [7.Define Parameter](#7.Define-Parameter)\n* [8.Best Parameter Search Using Bayesian](#8.Best-Parameter-Search-Using-Bayesian)\n* [9.Plotting results](#9.Plotting-results)\n* [10.Training LGB with Best Tuned Parameter](#10.Training-LGB-with-Best-Tuned-Parameter)\n* [11.Predict the results](#11.Predict-the-results)\n* [12.Final Submission](#12.Final-Submission)\n\n---\n\n\nInspired Kernel :  \n1) olivier's excellent [kernel](https://www.kaggle.com/ogrellier/plasticc-in-a-kernel-meta-and-data)  \n2) Siddhartha Bayesian Apprioach : [Kernel](https://www.kaggle.com/meaninglesslives/lgb-parameter-tuning/notebook?scriptVersionId=6733705)"},{"metadata":{"_uuid":"bce67b503f67cbce3b3c79200c700034684440d9"},"cell_type":"markdown","source":"## 1.Loading Important Library\n---"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport lightgbm as lgb\n\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.space import Real, Categorical, Integer\nfrom skopt.plots import plot_convergence\nfrom skopt.plots import plot_objective, plot_evaluations\nfrom skopt.utils import use_named_args\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e8d9c3d8180cc175b0f5f4fb203d15d8f0f1baf"},"cell_type":"markdown","source":"## 2.Helping Function\n---"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def lgb_multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n    \n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return 'wloss', loss, False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3636e89735c7415cb367866fda19955dd9deca49"},"cell_type":"code","source":"def multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b10269d3c0f0024596ce80fec38c63ccdd4dd6a"},"cell_type":"markdown","source":"## 3.Feature Extraction\n---\n### 1.Feature Extract from training set file"},{"metadata":{"trusted":true,"_uuid":"631c418da4275fba3070f2a6cff3873d11591f95"},"cell_type":"code","source":"gc.enable()\n\ntrain = pd.read_csv('../input/training_set.csv')\ntrain['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)\ntrain['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']\n\naggs = {\n    'mjd': ['min', 'max', 'size'],\n    'passband': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_err': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'detected': ['mean','std'],\n    'flux_ratio_sq':['sum','skew'],\n    'flux_by_flux_ratio_sq':['sum','skew'],\n}\n\nagg_train = train.groupby('object_id').agg(aggs)\nnew_columns = [\n    k + '_' + agg for k in aggs.keys() for agg in aggs[k]\n]\nagg_train.columns = new_columns\nagg_train['mjd_diff'] = agg_train['mjd_max'] - agg_train['mjd_min']\nagg_train['flux_diff'] = agg_train['flux_max'] - agg_train['flux_min']\nagg_train['flux_dif2'] = (agg_train['flux_max'] - agg_train['flux_min']) / agg_train['flux_mean']\nagg_train['flux_w_mean'] = agg_train['flux_by_flux_ratio_sq_sum'] / agg_train['flux_ratio_sq_sum']\nagg_train['flux_dif3'] = (agg_train['flux_max'] - agg_train['flux_min']) / agg_train['flux_w_mean']\n\ndel agg_train['mjd_max'], agg_train['mjd_min']\nagg_train.head()\n\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47fd417f5241d023733ef8361919c15aa6dc764e"},"cell_type":"markdown","source":"### 2.Feature Extract from training meta file\n---"},{"metadata":{"trusted":true,"_uuid":"1c6de093646141210e58362f9ea3b97f29fb170a"},"cell_type":"code","source":"meta_train = pd.read_csv('../input/training_set_metadata.csv')\nmeta_train.head()\n\nfull_train = agg_train.reset_index().merge(\n    right=meta_train,\n    how='outer',\n    on='object_id'\n)\n\nif 'target' in full_train:\n    y = full_train['target']\n    del full_train['target']\nclasses = sorted(y.unique())\n\n# Taken from Giba's topic : https://www.kaggle.com/titericz\n# https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n# with Kyle Boone's post https://www.kaggle.com/kyleboone\nclass_weight = {\n    c: 1 for c in classes\n}\nfor c in [64, 15]:\n    class_weight[c] = 2\n\nprint('Unique classes : ', classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b6f79b71d0e9b62266a52d9e7ac209c56e14aff"},"cell_type":"code","source":"if 'object_id' in full_train:\n    oof_df = full_train[['object_id']]\n    del full_train['object_id'], full_train['distmod'], full_train['hostgal_specz']\n    \n    \ntrain_mean = full_train.mean(axis=0)\nfull_train.fillna(train_mean, inplace=True)\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\nclfs = []\nimportances = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f86a8f970532b50956807173795c04691db110c9"},"cell_type":"markdown","source":"## 4.Parameter Tuning\n---"},{"metadata":{"trusted":true,"_uuid":"b2e76e21e2b4a5c86b6f624098ab43f6dd99131d"},"cell_type":"code","source":"dim_learning_rate = Real(low=1e-6, high=1e-1, prior='log-uniform',name='learning_rate')\ndim_estimators = Integer(low=50, high=2000,name='n_estimators')\ndim_max_depth = Integer(low=1, high=6,name='max_depth')\n\ndimensions = [dim_learning_rate,\n              dim_estimators,\n              dim_max_depth]\n\ndefault_parameters = [0.01,1500,4]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbf0c696e46d0335998d6522c1b35c522a67ed84"},"cell_type":"markdown","source":"## 5.Model Design"},{"metadata":{"trusted":true,"_uuid":"ac29bffcab6c94715687f2effe16b0f2ad575aae"},"cell_type":"code","source":"def createModel(learning_rate,n_estimators,max_depth):       \n\n    oof_preds = np.zeros((len(full_train), len(classes)))\n    for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n        trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n        val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n\n        clf = lgb.LGBMClassifier(**lgb_params,learning_rate=learning_rate,\n                                n_estimators=n_estimators,max_depth=max_depth)\n        clf.fit(\n            trn_x, trn_y,\n            eval_set=[(trn_x, trn_y), (val_x, val_y)],\n            eval_metric=lgb_multi_weighted_logloss,\n            verbose=False,\n            early_stopping_rounds=50\n        )\n        oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n        print('fold',fold_+1,multi_weighted_logloss(val_y, clf.predict_proba(val_x, num_iteration=clf.best_iteration_)))\n\n        clfs.append(clf)\n    \n    loss = multi_weighted_logloss(y_true=y, y_preds=oof_preds)\n    print('MULTI WEIGHTED LOG LOSS : %.5f ' % loss)\n    \n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79b47d0a29d997b00225dc8d1ae7da5f7becddc2"},"cell_type":"markdown","source":"## 6.Model Tuning"},{"metadata":{"trusted":true,"_uuid":"1e693c12f9a69b7bd23f38598979012aa257fb31"},"cell_type":"code","source":"@use_named_args(dimensions=dimensions)\ndef fitness(learning_rate,n_estimators,max_depth):\n    \"\"\"\n    Hyper-parameters:\n    learning_rate:     Learning-rate for the optimizer.\n    n_estimators:      Number of estimators.\n    max_depth:         Maximum Depth of tree.\n    \"\"\"\n\n    # Print the hyper-parameters.\n    print('learning rate: {0:.2e}'.format(learning_rate))\n    print('estimators:', n_estimators)\n    print('max depth:', max_depth)\n    \n    lv= createModel(learning_rate=learning_rate,\n                    n_estimators=n_estimators,\n                    max_depth = max_depth)\n    return lv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5be2d209b4bd083bf571c19aa8a5118141db31b8"},"cell_type":"markdown","source":"## 7.Define Parameter"},{"metadata":{"trusted":true,"_uuid":"c4eefae4eac16bd4cbf5e236e089505d21bf5ac1"},"cell_type":"code","source":"lgb_params = {\n    'boosting_type': 'gbdt',\n    'objective': 'multiclass',\n    'num_class': 14,\n    'metric': 'multi_logloss',\n    'subsample': .93,\n    'colsample_bytree': .75,\n    'reg_alpha': .01,\n    'reg_lambda': .01,\n    'min_split_gain': 0.01,\n    'min_child_weight': 10,\n    'silent':True,\n    'verbosity':-1,\n    'nthread':-1\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c17b738185e7d049111d9352d68101dceef34a80"},"cell_type":"code","source":"%%time\nerror = fitness(default_parameters)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b275e256cdc978d838499c592cd3aeee46bd85dc"},"cell_type":"markdown","source":"## 8.Best Parameter Search Using Bayesian"},{"metadata":{"trusted":true,"_uuid":"b0cb3ae266b3dde83fc282581de0628c60fae36a"},"cell_type":"code","source":"# use only if you haven't found out the optimal parameters for xgb. else comment this block.\nsearch_result = gp_minimize(func=fitness,\n                            dimensions=dimensions,\n                            acq_func='EI', # Expected Improvement.\n                            n_calls=20,\n                           x0=default_parameters)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8305dd7744e94ef2dedb08484dd278c9f21b39eb"},"cell_type":"markdown","source":"## 9.Plotting results"},{"metadata":{"trusted":true,"_uuid":"6654822f7a92bd9cccd3ef22bb9b3c6930ccd52f"},"cell_type":"code","source":"plot_convergence(search_result)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d91f31721a5a4674c093c701d9add7d922727302"},"cell_type":"code","source":"# optimal parameters found using scikit optimize. use these parameter to initialize the 2nd level model.\nprint(search_result.x)\nlearning_rate = search_result.x[0]\nn_estimators = search_result.x[1]\nmax_depth = search_result.x[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdc135548076463dab21ae122245783c5f5bfd11"},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\nclfs = []\nimportances = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d413e67b3a8735ceeec25b2c3fc6b911d8646464"},"cell_type":"markdown","source":"## 10.Training LGB with Best Tuned Parameter"},{"metadata":{"trusted":true,"_uuid":"4e5e1ed34e0887b3c10b8e649abef15f44a34853","scrolled":true},"cell_type":"code","source":"oof_preds = np.zeros((len(full_train), len(classes)))\nfor fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n    trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n    val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n    \n    clf = lgb.LGBMClassifier(**lgb_params,learning_rate=learning_rate,\n                                n_estimators=n_estimators,max_depth=max_depth)\n    clf.fit(\n        trn_x, trn_y,\n        eval_set=[(trn_x, trn_y), (val_x, val_y)],\n        eval_metric=lgb_multi_weighted_logloss,\n        verbose=100,\n        early_stopping_rounds=50\n    )\n    oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n    print(multi_weighted_logloss(val_y, clf.predict_proba(val_x, num_iteration=clf.best_iteration_)))\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = full_train.columns\n    imp_df['gain'] = clf.feature_importances_\n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    clfs.append(clf)\n\nprint('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_true=y, y_preds=oof_preds))\n\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))\nplt.tight_layout()\nplt.savefig('importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0caa1fc29c48dc0b35967cd9cad1eb3a306b7864"},"cell_type":"markdown","source":"## 11.Predict the results"},{"metadata":{"trusted":true,"_uuid":"1744237f1696ca45a697fb16b02515bdd49340dd"},"cell_type":"code","source":"meta_test = pd.read_csv('../input/test_set_metadata.csv')\n\nimport time\n\nstart = time.time()\nchunks = 5000000\nfor i_c, df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)):\n    df['flux_ratio_sq'] = np.power(df['flux'] / df['flux_err'], 2.0)\n    df['flux_by_flux_ratio_sq'] = df['flux'] * df['flux_ratio_sq']\n    # Group by object id\n    agg_test = df.groupby('object_id').agg(aggs)\n    agg_test.columns = new_columns\n    agg_test['mjd_diff'] = agg_test['mjd_max'] - agg_test['mjd_min']\n    agg_test['flux_diff'] = agg_test['flux_max'] - agg_test['flux_min']\n    agg_test['flux_dif2'] = (agg_test['flux_max'] - agg_test['flux_min']) / agg_test['flux_mean']\n    agg_test['flux_w_mean'] = agg_test['flux_by_flux_ratio_sq_sum'] / agg_test['flux_ratio_sq_sum']\n    agg_test['flux_dif3'] = (agg_test['flux_max'] - agg_test['flux_min']) / agg_test['flux_w_mean']\n\n    del agg_test['mjd_max'], agg_test['mjd_min']\n#     del df\n#     gc.collect()\n    \n    # Merge with meta data\n    full_test = agg_test.reset_index().merge(\n        right=meta_test,\n        how='left',\n        on='object_id'\n    )\n    full_test = full_test.fillna(train_mean)\n    \n    # Make predictions\n    preds = None\n    for clf in clfs:\n        if preds is None:\n            preds = clf.predict_proba(full_test[full_train.columns]) / folds.n_splits\n        else:\n            preds += clf.predict_proba(full_test[full_train.columns]) / folds.n_splits\n    \n   # Compute preds_99 as the proba of class not being any of the others\n    # preds_99 = 0.1 gives 1.769\n    preds_99 = np.ones(preds.shape[0])\n    for i in range(preds.shape[1]):\n        preds_99 *= (1 - preds[:, i])\n    \n    # Store predictions\n    preds_df = pd.DataFrame(preds, columns=['class_' + str(s) for s in clfs[0].classes_])\n    preds_df['object_id'] = full_test['object_id']\n    preds_df['class_99'] = 0.14 * preds_99 / np.mean(preds_99) \n    \n    if i_c == 0:\n        preds_df.to_csv('predictions.csv',  header=True, mode='a', index=False)\n    else: \n        preds_df.to_csv('predictions.csv',  header=False, mode='a', index=False)\n        \n    del agg_test, full_test, preds_df, preds\n    gc.collect()\n    \n    if (i_c + 1) % 10 == 0:\n        print('%15d done in %5.1f' % (chunks * (i_c + 1), (time.time() - start) / 60))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"664151cc04814f8f41d9ba699a55991403560499"},"cell_type":"markdown","source":"## 12.Final Submission"},{"metadata":{"trusted":true,"_uuid":"ab549b5d9fc196d47d2a5d7b73f241dda07cdd8c"},"cell_type":"code","source":"z = pd.read_csv('predictions.csv')\n\nprint(z.groupby('object_id').size().max())\nprint((z.groupby('object_id').size() > 1).sum())\n\nz = z.groupby('object_id').mean()\n\nz.to_csv('single_predictions.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e821ee754502dabaa2accf71a83095b32a259ab7"},"cell_type":"code","source":"##DONE","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}