{"cells":[{"metadata":{"_uuid":"656982859d470a3dd6b4da04916ea1b3ec84fa3d"},"cell_type":"markdown","source":"## The purpose of this kernel is to bring together features\n- The first 69 are from the 1.080 kernel which came via Oliver, Iprapas, and Chia-ta Tsai\n- Our Smote brought this to 1.052, implementing Scirpus 99 method brought to 1.039\n- Adding seven (7) features from our PB25 model brought to 1.030\n- Changing K from 5 to 12 brough to 1.029 (not recommended during development)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir('../input'))\nprint(os.listdir(\"../input/writefeaturetablefromsmotedartset\"))\nprint(os.listdir('../input/fork-of-aggregatecustomfeaturestest'))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ca4557fa813c935a5a4fcf6e2bcd86449aa0cc8"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport lightgbm as lgb\nfrom catboost import Pool, CatBoostClassifier\nimport itertools\nimport pickle, gzip\nimport glob\nfrom sklearn.preprocessing import StandardScaler\n#from tsfresh.feature_extraction import extract_features\nnp.warnings.filterwarnings('ignore')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b29f51e896671866d4b9872f410765f52e8c0a93"},"cell_type":"markdown","source":"## The next cells will be a bunch of functions brought in mostly from Chia-Ta Tsai's kernel\n- And he references Iprapas, Oliver, Kyle Boone, Giba\n- The Kaggle community shares a lot with one another\n- It has been challenging integrating their code with our functions\n- And it has forced us to learn / understand their code (to some degree)"},{"metadata":{"trusted":true,"_uuid":"458bb7b057c7d453c7f5acc33c70ff2646aabba5"},"cell_type":"code","source":"\"\"\"\n\nThis script is forked from chia-ta tsai's kernel of which he said:\n\nThis script is forked from iprapas's notebook \nhttps://www.kaggle.com/iprapas/ideas-from-kernels-and-discussion-lb-1-135\n\n#    https://www.kaggle.com/ogrellier/plasticc-in-a-kernel-meta-and-data\n#    https://www.kaggle.com/c/PLAsTiCC-2018/discussion/70908\n#    https://www.kaggle.com/meaninglesslives/simple-neural-net-for-time-series-classification\n#\n\"\"\"\n\nimport sys, os\nimport argparse\nimport time\nfrom datetime import datetime as dt\nimport gc; gc.enable()\nfrom functools import partial, wraps\n\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np # linear algebra\nnp.warnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom tsfresh.feature_extraction import extract_features\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52bba423f7570305957eb2b7236d20559ae27295"},"cell_type":"code","source":"\ndef multi_weighted_logloss(y_true, y_preds, classes, class_weights):\n    \"\"\"\n    refactor from\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2568f610eca18be4a298a5093bfe08ebddb844e9"},"cell_type":"code","source":"def lgbm_multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    refactor from\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"  \n    # Taken from Giba's topic : https://www.kaggle.com/titericz\n    # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n    # with Kyle Boone's post https://www.kaggle.com/kyleboone\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n\n    loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)\n    return 'wloss', loss, False\n\n\ndef xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):\n    loss = multi_weighted_logloss(y_true.get_label(), y_predicted, \n                                  classes, class_weights)\n    return 'wloss', loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e75adc9e2e0b62f8d155369fb7b26b1c496ea1af"},"cell_type":"code","source":"\ndef save_importances_archive(importances_):\n    mean_gain = importances_[['gain', 'feature']].groupby('feature').mean()\n    importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])\n    return importances_\n\ndef save_importances(importances_):\n    mean_gain = importances_[['gain', 'feature']].groupby('feature').mean()\n    importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])\n    plt.figure(figsize=(8, 12))\n    sns.barplot(x='gain', y='feature', data=importances_.sort_values('mean_gain', ascending=False))\n    plt.tight_layout()\n    plt.savefig('importances.png')\n    return importances_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd551479e01de426988e602ea4a716b6cad4f99e"},"cell_type":"code","source":"def xgb_modeling_cross_validation(params,\n                                  full_train, \n                                  y, \n                                  classes, \n                                  class_weights, \n                                  nr_fold=7, \n                                  random_state=1):\n    # Compute weights\n    w = y.value_counts()\n    weights = {i : np.sum(w) / w[i] for i in w.index}\n\n    # loss function\n    func_loss = partial(xgb_multi_weighted_logloss, \n                        classes=classes, \n                        class_weights=class_weights)\n\n    clfs = []\n    importances = pd.DataFrame()\n    folds = StratifiedKFold(n_splits=nr_fold, \n                            shuffle=True, \n                            random_state=random_state)\n    \n    oof_preds = np.zeros((len(full_train), np.unique(y).shape[0]))\n    for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n        trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n        val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n    \n        clf = XGBClassifier(**params)\n        clf.fit(\n            trn_x, trn_y,\n            eval_set=[(trn_x, trn_y), (val_x, val_y)],\n            eval_metric=func_loss,\n            verbose=100,\n            early_stopping_rounds=77,\n            sample_weight=trn_y.map(weights)\n        )\n        clfs.append(clf)\n\n        oof_preds[val_, :] = clf.predict_proba(val_x, ntree_limit=clf.best_ntree_limit)\n        print('no {}-fold loss: {}'.format(fold_ + 1, \n              multi_weighted_logloss(val_y, oof_preds[val_, :], \n                                     classes, class_weights)))\n    \n        imp_df = pd.DataFrame({\n                'feature': full_train.columns,\n                'gain': clf.feature_importances_,\n                'fold': [fold_ + 1] * len(full_train.columns),\n                })\n        importances = pd.concat([importances, imp_df], axis=0, sort=False)\n\n    score = multi_weighted_logloss(y_true=y, y_preds=oof_preds, \n                                   classes=classes, class_weights=class_weights)\n    print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))\n    df_importances = save_importances(importances_=importances)\n    df_importances.to_csv('xgb_importances.csv', index=False)\n    \n    return clfs, score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac952e9d027773b3e86cac4e1302769b22d4b4fb"},"cell_type":"markdown","source":"## This method is my main contribution"},{"metadata":{"trusted":true,"_uuid":"c38d90d50ea3b3a9d5f2b17c8a754670c7b42fe4"},"cell_type":"code","source":"\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.model_selection import train_test_split\nimport numpy as np # linear algebra\nimport pandas as pd\n\n#modify to work with kfold\n#def smoteAdataset(Xig, yig, test_size=0.2, random_state=0):\ndef smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):\n    \n        \n    sm=SMOTE(random_state=2)\n    Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())\n\n        \n    return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b355e13e6709dbd1e9ab34512b6a44f637a1555b"},"cell_type":"markdown","source":"## This is Oliver and Iprapas method but I integrated my Smote method into it"},{"metadata":{"trusted":true,"_uuid":"5694ec57ca9e922509f137b2bd34a790943158c0"},"cell_type":"code","source":"\ndef lgbm_modeling_cross_validation(params,\n                                   full_train, \n                                   y, \n                                   classes, \n                                   class_weights, \n                                   nr_fold=7, \n                                   random_state=1):\n\n    # Compute weights\n    w = y.value_counts()\n    weights = {i : np.sum(w) / w[i] for i in w.index}\n   # print(weights)\n   # weights=class_weights\n    clfs = []\n    importances = pd.DataFrame()\n    folds = StratifiedKFold(n_splits=nr_fold, \n                            shuffle=True, \n                            random_state=random_state)\n    \n    oof_preds = np.zeros((len(full_train), np.unique(y).shape[0]))\n    for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n        trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n        val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n        \n                \n        trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)\n        trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)\n    \n        val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)\n        \n        clf = LGBMClassifier(**params)\n        clf.fit(\n            trn_x, trn_y,\n            eval_set=[(trn_x, trn_y), (val_x, val_y)],\n            eval_metric=lgbm_multi_weighted_logloss,\n            verbose=100,\n            early_stopping_rounds=77,\n            sample_weight=trn_y.map(weights)\n        )\n        clfs.append(clf)\n\n        oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n        print('no {}-fold loss: {}'.format(fold_ + 1, \n              multi_weighted_logloss(val_y, oof_preds[val_, :], \n                                     classes, class_weights)))\n    \n        imp_df = pd.DataFrame({\n                'feature': full_train.columns,\n                'gain': clf.feature_importances_,\n                'fold': [fold_ + 1] * len(full_train.columns),\n                })\n        importances = pd.concat([importances, imp_df], axis=0, sort=False)\n\n    score = multi_weighted_logloss(y_true=y, y_preds=oof_preds, \n                                   classes=classes, class_weights=class_weights)\n    print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))\n    df_importances = save_importances(importances_=importances)\n    df_importances.to_csv('lgbm_importances.csv', index=False)\n    \n    return clfs, score, oof_preds, importances","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26f04afe8949abeec40d9ba47cd2c06931592859"},"cell_type":"markdown","source":"## I made modifications to these methods (predict_chunk, process_test)\n- Separating feature extraction from modeling potentially allows us to do more within the 6 hour kernel window\n- It allows us to generate features on parallel paths\n- The chunk processing still seemed necessary as the kernel timed out without it"},{"metadata":{"trusted":true,"_uuid":"4c857770dd12b0c64c9bf9caab5576480e687df4"},"cell_type":"code","source":"\ndef predict_chunk(df_, clfs_, features, train_mean):\n    # Group by object id    \n    agg_ = df_\n    # Merge with meta data\n    full_test = agg_.reset_index()\n    print(full_test.head())\n\n    full_test = full_test.fillna(0)\n    # Make predictions\n    preds_ = None\n    for clf in clfs_:\n        if preds_ is None:\n            preds_ = clf.predict_proba(full_test[features]) / len(clfs_)\n        else:\n            preds_ += clf.predict_proba(full_test[features]) / len(clfs_)\n            \n    #going to recalc 99 below anyways\n    # Compute preds_99 as the proba of class not being any of the others\n    # preds_99 = 0.1 gives 1.769\n    preds_99 = np.ones(preds_.shape[0])\n    \n    \n    for i in range(preds_.shape[1]):\n        preds_99 *= (1 - preds_[:, i])\n\n    # Create DataFrame from predictions\n    preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s) for s in clfs_[0].classes_])\n    preds_df_['object_id'] = full_test['object_id']\n    preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99) \n\n    return preds_df_","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"803854aecae354d60c2537d594ebd6b53e471e5d"},"cell_type":"markdown","source":"## The filenaming here is counter-intuitive\n- I ran into memory issues and the approach I took became necessary (or at least practical)\n- I write testdf to a file that will later be overwritten by the predictions\n- During the loop the features are being read from filename (which is like subm%)\n- During the loop the predictions are being written to predictions.csv\n- At the end the prediction dataFrame overwrites the featureSet file"},{"metadata":{"trusted":true,"_uuid":"0f32f3df59692561373a83fc08ef7768cc5f9ce3"},"cell_type":"code","source":"def process_test(clfs, \n                 testdf,\n                 full_train,\n                 train_mean,\n                 filename='submission.csv',\n                 chunks=40615):\n\n    import time\n    \n    #choose a value for chunks such that testdf.shape[0]%chunks=0 - this saves you headache at the end\n    start = time.time()\n    chunks = 40615\n    # df=df.round(8)\n    testdf=testdf.round(4)\n    testdf.to_csv(filename, index=False)\n    for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)):\n\n        print(df.shape)\n        preds_df = predict_chunk(df_=df,\n                                 clfs_=clfs,\n                                 features=full_train.columns,\n                                 train_mean=train_mean)\n\n        if i_c == 0:\n            preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)\n        else:\n            preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)\n\n        del preds_df\n        gc.collect()\n\n        print('%15d done in %5.1f minutes' % (chunks * (i_c + 1), (time.time() - start) / 60), flush=True)\n\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70a5ef4ba2d242a4470ebcb759547517632682f8"},"cell_type":"code","source":"#from Scirpus discussion:\n\ndef GenUnknown(data):\n    return ((((((data[\"mymedian\"]) + (((data[\"mymean\"]) / 2.0)))/2.0)) + (((((1.0) - (((data[\"mymax\"]) * (((data[\"mymax\"]) * (data[\"mymax\"]))))))) / 2.0)))/2.0)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d65a7023d08e79e2585e60cb05b7bcaf4ef64e9"},"cell_type":"code","source":"best_params = {\n            'device': 'cpu', \n            'objective': 'multiclass', \n            'num_class': 14, \n            'boosting_type': 'gbdt', \n            'n_jobs': -1, \n            'max_depth': 7, \n            'n_estimators': 1000, #was 500\n            'subsample_freq': 2, \n            'subsample_for_bin': 5000, \n            'min_data_per_group': 200, #was 100\n            'max_cat_to_onehot': 4, \n            'cat_l2': 1.0, \n            'cat_smooth': 59.5, \n            'max_cat_threshold': 32, \n            'metric_freq': 10, \n            'verbosity': -1, \n            'metric': 'multi_logloss', \n            'xgboost_dart_mode': False, \n            'uniform_drop': False, \n            'colsample_bytree': 0.5, \n            'drop_rate': 0.173, \n            'learning_rate': 0.0133, #was .267\n            'max_drop': 5, \n            'min_child_samples': 20, #was 10\n            'min_child_weight': 100.0, \n            'min_split_gain': 0.1, \n            'num_leaves': 7, \n            'reg_alpha': 0.1, \n            'reg_lambda': 0.00023, \n            'skip_drop': 0.44, \n            'subsample': 0.75}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e3f26aa91f757d8258c9f6be2cb264fc07c68d8"},"cell_type":"markdown","source":"## The cells below are adapted from Chai-Ta Tsai's 'main' method\n- I made changes to enable usage of my features\n- If you add features, make sure you add the same features to both testdf and full_train\n- Be aware of memory issues.  It may be necessary to delete some sets once they've been used.  "},{"metadata":{"trusted":true,"_uuid":"925d35e22e6c762de3e87533c87b9a2e7db90613"},"cell_type":"code","source":"#Here is a change from the script\n#training features\ntrainingDartDf=pd.read_csv('../input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')\ntrainingJimsDf=pd.read_csv('../input/aggregatecustomfeatures/trainingAggregate.csv')\nif 'Unnamed: 0' in trainingDartDf.columns:\n    trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)\nprint(trainingDartDf.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba8a0a15b64cfbedd1dc2bc125ea9f344710efee"},"cell_type":"code","source":"\n#trainingJimsDf=aggForTwoPb(trainingJimsDf, 2, 5)\nprint(trainingJimsDf.shape)\ntrainingJimsDf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e68ae8fced7ce0db6f729d8ebfdc1fba0d6b4364"},"cell_type":"code","source":"trainingJimsDf.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8eb7c4e7b43ee666a0b2bfa6d5281bb3294be0f"},"cell_type":"code","source":"#trainingDartDf.head()\ncolumnsToAdd=['pb0maxstd', 'pb0medmed', 'pb0maxavg', 'pb0maxdsl',\n       'pb3maxstd', 'pb3medmed', 'pb3maxavg', 'pb3maxdsl', 'pb3transitory',\n       'pb0transitory', 'pb1maxstd', 'pb1medmed', 'pb1maxavg', 'pb1maxdsl',\n       'pb4maxstd', 'pb4medmed', 'pb4maxavg', 'pb4maxdsl', 'pb4transitory',\n       'pb1transitory', 'pb2maxstd', 'pb2medmed', 'pb2maxavg', 'pb2maxdsl',\n       'pb5maxstd', 'pb5medmed', 'pb5maxavg', 'pb5maxdsl', 'pb5transitory',\n       'pb2transitory', 'irmaxstd', 'vismaxstd', 'uvmaxstd', 'irmedmed',\n       'vismedmed', 'uvmedmed', 'irmaxavg', 'vismaxavg', 'uvmaxavg',\n       'irmaxdsl', 'vismaxdsl', 'uvmaxdsl', 'irtransitory', 'vistransitory',\n       'uvtransitory', 'irSpread', 'uvSpread', 'visSpread', 'irUvSpreadRatio',\n       'irMinusUvTransitory', 'irVisSpreadRatio', 'irMinusVisTransitory']\n\nfor column in columnsToAdd:\n    trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]\n\ntraindf=trainingDartDf\n\n#from the 1.052 kernel\ndel traindf['hostgal_specz']\ndel traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']\ndel traindf['ddf']\n\n#these appear to either be properties of the measurement system or collinear with the newly added features\n#del traindf['flux_err_mean'], traindf['flux_err_median'],traindf['flux_err_std'],traindf['flux_err_max'], traindf['flux_err_min']\n#del traindf['flux_mean'], traindf['flux_std'], traindf['flux_max']\n\n\nprint(traindf.shape)\ntraindf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa605f62c429d9f85b1b1a60da8bda7b3baf2ac4"},"cell_type":"code","source":"    #test features\n    testDartDf=pd.read_csv('../input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')\n    testJimsDf=pd.read_csv('../input/fork-of-aggregatecustomfeaturestest/testAggregate.csv')\n\n    if 'Unnamed: 0' in testDartDf.columns:\n        testDartDf=testDartDf.drop('Unnamed: 0', axis=1)\n    print(testDartDf.shape)\n    testDartDf.head()\n\n    for column in columnsToAdd:\n        testDartDf[column]=testJimsDf[column]\n\n    testdf=testDartDf\n\n    #from the 1.052 kernel\n    del testdf['hostgal_specz']\n    del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']\n    del testdf['ddf']\n\n    testdf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"727a9cfef5c4f9fe34cb0290a5db3bb4c0ae1933"},"cell_type":"code","source":"full_train=traindf\nif 'target' in full_train:\n    y = full_train['target']\n    del full_train['target']\n\nclasses = sorted(y.unique())    \n# Taken from Giba's topic : https://www.kaggle.com/titericz\n# https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n# with Kyle Boone's post https://www.kaggle.com/kyleboone\nclass_weights = {c: 1 for c in classes}\nclass_weights.update({c:2 for c in [64, 15]})\nprint('Unique classes : {}, {}'.format(len(classes), classes))\nprint(class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed28379fd5769a75cd923f776cfad3b2c25dc519"},"cell_type":"code","source":"\nif 'object_id' in full_train:\n    oof_df = full_train[['object_id']]\n    del full_train['object_id'] \n    #del full_train['distmod'] \n\ntrain_mean = full_train.mean(axis=0)\n#train_mean.to_hdf('train_data.hdf5', 'data')\npd.set_option('display.max_rows', 500)\n#print(full_train.describe().T)\n#import pdb; pdb.set_trace()\nfull_train.fillna(0, inplace=True)\nprint(full_train.shape)\nfull_train.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d7cd750139dfd15b412f7092d5d71691357f83c"},"cell_type":"code","source":"print(class_weights)\nprint(classes)\nprint(y.shape)\nprint(full_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cceeff1b71b57711518450c09521679fa4bfada9"},"cell_type":"markdown","source":"## The first two lines (or lack thereof) have caused me more headache than I can count\n- it has to do with numpy data types when native data types are expected"},{"metadata":{"_uuid":"e55a7594a01e9fe511598ea4b2d1f1f478a81747"},"cell_type":"markdown","source":"## CV score (multi-weighted log loss at the end of the output) is a good predictor of LB score\n- Changing the number of folds improves CV a lot without much impact on LB, so leave it at 5\n- Things that are likely to induce overfitting will increase the delta between CV and LB\n- Right now the mapping we've seen with the surrounding code is 0.7 --> 1.110, 0.649 --> 1.039, 0.638 --> 1.030\n- Our latest estimate is LB = CV + 0.392\n- baseline with first seven features is .638"},{"metadata":{"trusted":true,"_uuid":"829a1e13cd34498acbacd4101ea7a75c2d2cf9c9","scrolled":true},"cell_type":"code","source":"for cindex in full_train.columns:\n    full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])\n\neval_func = partial(lgbm_modeling_cross_validation, \n                        full_train=full_train, \n                        y=y, \n                        classes=classes, \n                        class_weights=class_weights, \n                        nr_fold=5, \n                        random_state=1)\n\nbest_params.update({'n_estimators': 2000}) #was 1000\n    \n    # modeling from CV\nclfs, score, oof_preds, importances = eval_func(best_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52cb160f9abf40638c7a77585b95a54e721657eb"},"cell_type":"code","source":"from matplotlib import pyplot as plt\nsave_importances(importances_=importances)\n# http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    \nunique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n        \ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\n\n# Compute confusion matrix\nfrom sklearn.metrics import confusion_matrix\ncnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1))\nnp.set_printoptions(precision=2)\n\nsample_sub = pd.read_csv('../input/PLAsTiCC-2018/sample_submission.csv')\nclass_names = list(sample_sub.columns[1:-1])\ndel sample_sub;gc.collect()\n\n# Plot non-normalized confusion matrix\nplt.figure(figsize=(12,12))\nfoo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True,\n                      title='Confusion matrix')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e10efd2330796c0bf1dad9f61ab61beeadd87f6f"},"cell_type":"code","source":"\nfilename = 'subm_{:.6f}_{}.csv'.format(score, \n                 dt.now().strftime('%Y-%m-%d-%H-%M'))\nprint('save to {}'.format(filename))\n# TEST\n\n\nprocess_test(clfs, \n             testdf,\n             full_train,\n             train_mean=train_mean, \n             filename=filename,\n             chunks=40615)\n\n\npdf = pd.read_csv('predictions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a52038566d64b21ba8ec5fd954b95b5643cc3731"},"cell_type":"code","source":"\n\n\n# get a list of columns\ncols = list(pdf)\n# move the column to head of list using index, pop and insert\ncols.insert(0, cols.pop(cols.index('object_id')))\npdf = pdf.loc[:, cols]\n\n\n\nfeats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',\n         'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',\n         'class_92', 'class_95']\n\ny = pd.DataFrame()\ny['mymean'] = pdf[feats].mean(axis=1)\ny['mymedian'] = pdf[feats].median(axis=1)\ny['mymax'] = pdf[feats].max(axis=1)\n\npdf['class_99'] = GenUnknown(y)\n\npdf.to_csv(filename, index=False)\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}