{"cells":[{"metadata":{"_uuid":"d4d1eb99a9ba3f3550f520bd8b7442dd2642a6fc"},"cell_type":"markdown","source":"# What's New"},{"metadata":{"_uuid":"a4d5742f14d7d32a8d317f41bbb8a5b577c814ce"},"cell_type":"markdown","source":"These are some of the simple ideas that were not present in my previous public kernel\n\n*  A better NN Architecture inspired by DenseNet\n\n*  A better feature pre-processing method (PowerTransformer) for NN.\n\n*  Post processing the predicted probabilities\n\nSome other useful ideas (that i am not including here to avoid making the kernel messy)\n\n* Resampling the time series\n\n    I found this to be really useful for training neural networks. You can get a boost of around +0.01 if you decide to use this.\n\n* Second level modelling\n\n    You can stack the predictions from nn with other methods like lgb,xgb and get very good boost. \n    \n* A custom loss\n\n    You can get some interesting insights from analysing the confusion matrix and design a custom loss accordingly. For example class 90 keeps getting confused with class 42 and 52. You can try to suppress such predictions.\n\n* Neural Network weight initialization\n\n    You can try experimenting with different weight initialization schemes. It helps in faster convergance and in finding a better minima.\n\n\n\n"},{"metadata":{"_uuid":"2a2e4540aa68edf7b05dc0393e8cee68b568f652"},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport lightgbm as lgb\nfrom catboost import Pool, CatBoostClassifier\nimport itertools\nimport pickle, gzip\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def lgb_multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n    \n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return 'wloss', loss, False","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd440bf5e924a73e6fd8d1e94293b16828a155cb"},"cell_type":"markdown","source":"# Aggregating features globally (i.e. considering all passbands) and passband wise."},{"metadata":{"trusted":true,"_uuid":"631c418da4275fba3070f2a6cff3873d11591f95"},"cell_type":"code","source":"gc.enable()\n\ntrain = pd.read_csv('../input/PLAsTiCC-2018/training_set.csv')\n\ntrain['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)\ntrain['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']\n\naggs = {\n    'flux': ['min', 'max', 'mean', 'median', 'std','sum','skew'],\n    'flux_err': ['min', 'max', 'mean','skew'],\n    'detected': [ 'mean', 'std','sum'],\n    'flux_ratio_sq':['mean','sum','skew'],\n    'flux_by_flux_ratio_sq':['mean','sum','skew'],\n}\n\naggs_global = {\n    'mjd': ['size'],\n    'flux': ['min', 'max', 'mean', 'median', 'std','sum','skew'],\n    'flux_err': ['min', 'max', 'mean', 'median', 'std','sum','skew'],\n    'detected': [ 'mean','skew','median','sum'],\n    'flux_ratio_sq':['min', 'max', 'mean','sum','skew'],\n    'flux_by_flux_ratio_sq':['min', 'max', 'mean','sum','skew'],\n}\n\nagg_train_global_feat = train.groupby('object_id').agg(aggs_global)\n\nnew_columns = [\n    k + '_' + agg for k in aggs.keys() for agg in aggs[k]\n]\n\nnew_columns_global = [\n    k + '_' + agg for k in aggs_global.keys() for agg in aggs_global[k]\n]\n\nagg_train_global_feat.columns = new_columns_global\n\nagg_train = train.groupby(['object_id','passband']).agg(aggs)\n\nagg_train = agg_train.unstack()\n\ncol_names = []\nfor col in new_columns:\n    for i in range(6):\n        col_names.append(col+'_'+str(i))\n        \nagg_train.columns = col_names\nagg_train_global_feat['flux_diff'] = agg_train_global_feat['flux_max'] - agg_train_global_feat['flux_min']\nagg_train_global_feat['flux_dif2'] = (agg_train_global_feat['flux_max'] - agg_train_global_feat['flux_min']) / agg_train_global_feat['flux_mean']\nagg_train_global_feat['flux_w_mean'] = agg_train_global_feat['flux_by_flux_ratio_sq_sum'] / agg_train_global_feat['flux_ratio_sq_sum']\nagg_train_global_feat['flux_dif3'] = (agg_train_global_feat['flux_max'] - agg_train_global_feat['flux_min']) / agg_train_global_feat['flux_w_mean']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1004bae7e3fc3a00e8ac5be793b4163338c5de7a"},"cell_type":"markdown","source":"# Computing [Sionkowski's Feature](https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69696#410538)"},{"metadata":{"trusted":true,"_uuid":"e8439724a568ee9084e49472b0543bee48f3e149"},"cell_type":"code","source":"# Legacy code. There are much better ways to compute this but for train set this suffices so \n# i got too lazy to change https://www.kaggle.com/c/PLAsTiCC-2018/discussion/71398\ndef detected_max(mjd,detected):\n    try:     return max(mjd[detected==1]) - min(mjd[detected==1])\n    except:  return 0\n    \ntemp = train.groupby('object_id').apply(lambda x:detected_max(x['mjd'],x['detected']))\ntemp1 = train.groupby(['object_id','passband']).apply(lambda x:detected_max(x['mjd'],x['detected'])).unstack()\ntemp.columns = ['mjd_global_diff']\ntemp1.columns = ['mjd_pb0','mjd_pb1','mjd_pb2','mjd_pb3','mjd_pb4','mjd_pb5']\ntemp = temp.reset_index()\ntemp1 = temp1.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5c70e692717be7f14294d26adcda6db33b11dfd"},"cell_type":"markdown","source":"# Aggregating features only on detected events"},{"metadata":{"trusted":true,"_uuid":"bc8b4b6788df563472c52806325058156ec1daf1"},"cell_type":"code","source":"aggs_det = {\n    'flux': ['min','mean', 'max','skew'],\n    'flux_ratio_sq':['min','mean', 'max','skew'],\n    'flux_by_flux_ratio_sq':['min', 'max','mean','skew'],\n}\n\ntrain_detected =  train[train.detected==1]\ntemp2 = train_detected.groupby(['object_id']).agg(aggs_det)\n       \nnew_columns_det = [\n    k + '_det_' + agg for k in aggs_det.keys() for agg in aggs_det[k]\n]\n\ntemp2.columns = new_columns_det\ntemp2['flux_diff_det'] = temp2['flux_det_max'] - temp2['flux_det_min']\ntemp2['flux_ratio_sq_diff_det'] = temp2['flux_ratio_sq_det_max'] - temp2['flux_ratio_sq_det_min']\ntemp2['flux_by_flux_ratio_sq_diff_det'] = temp2['flux_by_flux_ratio_sq_det_max'] - temp2['flux_by_flux_ratio_sq_det_min']\n\ndel temp2['flux_by_flux_ratio_sq_det_max'],temp2['flux_by_flux_ratio_sq_det_min']\ndel temp2['flux_ratio_sq_det_max'],temp2['flux_ratio_sq_det_min']\ndel temp2['flux_det_max'],temp2['flux_det_min']\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f547214a1889ab64533ef8b4a697fe4f2b4e030"},"cell_type":"markdown","source":"# Merging all the aggregated features"},{"metadata":{"trusted":true,"_uuid":"3b36533fb25eadaa8ed03cbac46048671adae7a1"},"cell_type":"code","source":"meta_train = pd.read_csv('../input/PLAsTiCC-2018/training_set_metadata.csv')\n\nfull_train = agg_train.reset_index().merge(\n    right=meta_train,\n    how='outer',\n    on='object_id'\n)\n\nfull_train = full_train.merge(\n    right=agg_train_global_feat,\n    how='outer',\n    on='object_id'\n)\n\nfull_train = full_train.merge(\n    right=temp,\n    how='outer',\n    on='object_id'\n)\n\nfull_train = full_train.merge(\n    right=temp1,\n    how='outer',\n    on='object_id'\n)\n\nfull_train = full_train.merge(\n    right=temp2,\n    how='outer',\n    on='object_id'\n)\n\nif 'target' in full_train:\n    y = full_train['target']\n    del full_train['target']\nclasses = sorted(y.unique())\n\nclass_weight = {\n    c: 1 for c in classes\n}\nfor c in [64, 15]:\n    class_weight[c] = 2\n\nprint('Unique classes : ', classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b6f79b71d0e9b62266a52d9e7ac209c56e14aff"},"cell_type":"code","source":"if 'object_id' in full_train:\n    oof_df = full_train[['object_id']]\n    del full_train['object_id'], full_train['hostgal_specz']\n    del full_train['ra'], full_train['decl'], full_train['gal_l'],full_train['gal_b'],full_train['ddf']","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"415bfec3d0c9143364c82d3ae82cd3a93634855c"},"cell_type":"code","source":"# # one possible way to do resampling\n# train = pd.read_csv('../input/PLAsTiCC-2018/training_set.csv')\n# # train = train.sample(frac=0.6)\n# train['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)\n# train['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']\n# train.set_index('object_id',inplace=True)\n# train_sampled = [train.loc[obj_id,:].sample(frac=random.uniform(0.3,0.7)) for obj_id in train.index.unique()]\n# train = pd.concat(train_sampled,axis=0)\n# train = train.reset_index()\n# train.sort_values( ['object_id', 'mjd'], ascending=True, inplace=True )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5467a310dee5adcdbd9e17535eaf8f18b62dc20"},"cell_type":"markdown","source":"# Eliminating the least useful features based on feature importance"},{"metadata":{"trusted":true,"_uuid":"6c534e753b40fb41af91f53674684cb96fcf2fb9"},"cell_type":"code","source":"useless_cols = [\n'flux_max_2',\n'flux_median_0',\n'flux_median_4',\n'flux_err_skew_1',\n'flux_err_skew_3',\n'detected_mean_4',\n'detected_std_3',\n'detected_std_4',\n'detected_sum_4',\n'flux_ratio_sq_mean_4',\n'flux_ratio_sq_sum_3',\n'flux_ratio_sq_sum_4',\n'flux_median',\n'flux_err_skew',\n'flux_ratio_sq_sum',\n'mjd_pb5',\n'flux_ratio_sq_det_skew',\n]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be1d8f864de5f6a42530597cc2f26df0aecc3b8c"},"cell_type":"markdown","source":"# Preprocessing using PowerTransformer"},{"metadata":{"_uuid":"b0db74acf6e6232851245a279a113e3ddc7cf091"},"cell_type":"markdown","source":"For neural networks, preprocessing the data is** extremely important**. I tried all the available preprocessing methods available in sklearn and some other methods also. I found PowerTransform and GaussRank Scaler to be the best methods. Power transforms helps to make the data more Gaussian like.  For more details  [refer](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PowerTransformer.html)."},{"metadata":{"trusted":true,"_uuid":"266166876c95f544d6bbbaab61671956e612571d"},"cell_type":"code","source":"full_train_new = full_train.drop(useless_cols,axis=1)\nfrom sklearn.preprocessing import PowerTransformer\nss = PowerTransformer()\nfull_train_ss = ss.fit_transform(np.nan_to_num(full_train_new))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b7424590dc985a79354838aa99ed5cb94ebc3f7"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense,BatchNormalization,Dropout\nfrom keras.callbacks import ReduceLROnPlateau,ModelCheckpoint\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nimport keras\nfrom keras import regularizers\nfrom collections import Counter\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12e0ed6fb47608f02a5447ba3343aea55460d86c"},"cell_type":"markdown","source":"# Loss Function\n\nFocal Loss is generally useful for imbalanced classes . You can find more details [here](https://arxiv.org/abs/1708.02002) . If you use it for training it leads to higher accuracy but the lb score will be slightly lower since we are not directly optimizing the actual objective.\n"},{"metadata":{"trusted":true,"_uuid":"13118fe2b9c9e163b7793c20cb713dfcfc310f1f"},"cell_type":"code","source":"def focal_loss(gamma=2., alpha=.25):\n    def focal_loss_fixed(y_true, y_pred):\n        yc = tf.clip_by_value(y_pred,1e-15,1-1e-15)\n        pt_1 = tf.where(tf.equal(y_true, 1), yc, tf.ones_like(yc))\n        pt_0 = tf.where(tf.equal(y_true, 0), yc, tf.zeros_like(yc))\n        return (-K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1))-K.sum((1-alpha) * K.pow( pt_0, gamma) * K.log(1. - pt_0)))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59889ae9a9041a1c9efa8c42c59b75aa58249bf1"},"cell_type":"code","source":"def mywloss(y_true,y_pred):  \n    yc=tf.clip_by_value(y_pred,1e-15,1-1e-15)\n    loss=-(tf.reduce_mean(tf.reduce_mean(y_true*tf.log(yc),axis=0)/wtable))\n#     a sample custom loss\n#     loss=-2*(tf.reduce_mean(tf.reduce_mean(y_true*tf.log(yc),axis=0)/wtable_new)) \\\n#     + 0.5*tf.reduce_mean(tf.reduce_mean((1-y_true[:,11])*yc[:,11],axis=0)) + 0.10*tf.reduce_mean(tf.reduce_mean((1-y_true[:,6])*yc[:,6],axis=0))\n#     + 0.25*tf.reduce_mean(tf.reduce_mean((1-y_true[:,9])*yc[:,9],axis=0)) + 0.1*tf.reduce_mean(tf.reduce_mean((1-y_true[:,4])*yc[:,4],axis=0))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c35b11a75cdc6aa926109ae876a1f2bb9f4078c"},"cell_type":"code","source":"def multi_weighted_logloss(y_ohe, y_p):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9235d7edc1520d10784f0f14b88e35651d7c9d5b"},"cell_type":"code","source":"unique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n    \norig_class_map = dict()\nfor i,val in enumerate(unique_y):\n    orig_class_map[i] = val\n    \ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\ny_categorical = to_categorical(y_map)\n\ny_count = Counter(y_map)\nwtable = np.zeros((len(unique_y),))\nfor i in range(len(unique_y)):\n    wtable[i] = y_count[i]/y_map.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449a0c6c0f3baa75ef4719ef8e76ac5fa62d094b"},"cell_type":"code","source":"def plot_loss_acc(history):\n    plt.figure(figsize=(20,7))\n    plt.subplot(1,2,1)\n    plt.plot(history.history['loss'][1:])\n    plt.plot(history.history['val_loss'][1:])\n    plt.title('model loss')\n    plt.ylabel('val_loss')\n    plt.xlabel('epoch')\n    plt.legend(['Train','Validation'], loc='upper left')\n    \n    plt.subplot(1,2,2)\n    plt.plot(history.history['acc'][1:])\n    plt.plot(history.history['val_acc'][1:])\n    plt.title('Model Accuracy')\n    plt.ylabel('val_acc')\n    plt.xlabel('epoch')\n    plt.legend(['Train','Validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b979aa9078e50523f13faa5a8b91ea1937aafc21"},"cell_type":"code","source":"from keras.layers import Input,Dense,Conv1D,MaxPool1D,GlobalMaxPooling1D,Add,GlobalAveragePooling1D,Reshape,multiply\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras import callbacks","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdd6690ae64781e66bcc559fb5b44cb15698406c"},"cell_type":"markdown","source":"A slightly better keras architecture inspired by DenseNet\n\n The last layer has 3 inputs, consisting of the feature maps of all preceding convolutional blocks. "},{"metadata":{"trusted":true,"_uuid":"da06f38cb32e4415e9d12e618681891fc224d1b2"},"cell_type":"code","source":"K.clear_session()\ndef build_model(dropout_rate=0.25,activation='relu'):\n    start_neurons = 256\n    feat_ip = Input(shape=(full_train_ss.shape[1],), name='feature_ip')\n    \n    x = BatchNormalization()(feat_ip)\n    x = Dense(start_neurons, activation=activation)(x)    \n    feat1 = Dropout(rate=dropout_rate)(x)\n\n    x = BatchNormalization()(feat1)\n    x = Dense(start_neurons//2, activation=activation)(x)    \n    feat2 = Dropout(rate=dropout_rate)(x)\n    \n    x = BatchNormalization()(feat2)\n    x = Dense(start_neurons//4, activation=activation)(x)    \n    feat3 = Dropout(rate=dropout_rate)(x)\n    \n    x = BatchNormalization()(feat3)\n    x = Dense(start_neurons//8, activation=activation)(x)    \n    feat4 = Dropout(rate=dropout_rate/2)(x)\n\n    feat_concat = concatenate([feat1,feat2,feat3,feat4])\n    out = Dense(len(classes), activation='softmax')(feat_concat)\n\n    model = Model(inputs=[feat_ip], outputs=[out])    \n    return model   ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1d699a8e9d0911dbbdfb419ef1e50a63bf4a3e0"},"cell_type":"markdown","source":"# Cosine Annealing for learning rate scheduling"},{"metadata":{"_uuid":"6b9fad982a9a7bc338c368b884dca9d1f4eec1d5"},"cell_type":"markdown","source":"I found cosine annealing to be slightly better than ReduceLROnPlateau. You can even use it for creating different [snapshots](http://openreview.net/pdf?id=BJYwwY9ll) of the model. "},{"metadata":{"trusted":true,"_uuid":"afd7d863933406d7f19135867d9188663d0d0718"},"cell_type":"code","source":"# https://github.com/titu1994/Snapshot-Ensembles\nclass SnapshotCallbackBuilder:\n    def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):\n        self.T = nb_epochs\n        self.M = nb_snapshots #for single model it should be 1\n        self.alpha_zero = init_lr\n\n    def get_callbacks(self, model_prefix='Model'):\n\n        callback_list = [\n            ModelCheckpoint(\"./keras.model\",monitor='val_loss',mode = 'min', save_best_only=True, verbose=0),\n            callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)\n        ]\n\n        return callback_list\n\n    def _cosine_anneal_schedule(self, t):\n        cos_inner = np.pi * (t % (self.T // self.M))  # t - 1 is used when t has 1-based indexing.\n        cos_inner /= self.T // self.M\n        cos_out = np.cos(cos_inner) + 1\n        return float(self.alpha_zero / 2 * cos_out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6417136e31687df52d5d9f85a83df00aaa6cf2eb","scrolled":true},"cell_type":"code","source":"clfs = []\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\noof_preds = np.zeros((len(full_train_ss), len(classes)))\nepochs = 250\nbatch_size = 200\nfor fold_, (trn_, val_) in enumerate(folds.split(y_map, y_map)):\n\n    checkPoint = ModelCheckpoint(\"./keras.model\",monitor='val_loss',mode = 'min', save_best_only=True, verbose=0)\n    x_train, y_train = full_train_ss[trn_], y_categorical[trn_]\n    x_valid, y_valid = full_train_ss[val_], y_categorical[val_]\n    \n    model = build_model(dropout_rate=0.5,activation='tanh')    \n    # Compile model    \n    model.compile(loss=mywloss, optimizer='adam', metrics=['accuracy'])\n    snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)\n    history = model.fit(x_train, y_train,\n                    validation_data=[x_valid, y_valid], \n                    epochs=epochs,\n                    batch_size=batch_size,shuffle=True,verbose=0,callbacks=[checkPoint])       \n    \n    plot_loss_acc(history)\n    \n    print('Loading Best Model')\n    model.load_weights('./keras.model')\n    # # Get predicted probabilities for each class\n    oof_preds[val_, :] = model.predict(x_valid,batch_size=batch_size)\n    print(multi_weighted_logloss(y_valid, model.predict(x_valid,batch_size=batch_size)))\n    clfs.append(model)\n    \nprint('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_categorical,oof_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8d149b6795271bdce59ee091118bd1681cb3462"},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc5c23f2780db663e813e796b811f3831ef42686"},"cell_type":"code","source":"# Compute confusion matrix\ncnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1))\nnp.set_printoptions(precision=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"161c3e5e83a2ee3927df2c782fcb53fb7a379292"},"cell_type":"code","source":"test_files1 = glob.glob('../input/create-test-set/*.gz')\nsample_sub = pd.read_csv('../input/PLAsTiCC-2018/sample_submission.csv')\nclass_names = list(sample_sub.columns[1:-1])\ndel sample_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9416f8a1b8b43e49f54642e03793ffce83adedd","scrolled":true},"cell_type":"code","source":"# Plot non-normalized confusion matrix\nplt.figure(figsize=(7,7))\nfoo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True,\n                      title='Confusion matrix, without normalization')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f65492856178f1692de22989481d7dde8afd1e84"},"cell_type":"markdown","source":"# Test Set Predictions"},{"metadata":{"_uuid":"055ae1cdef8ae1c7e56881c3577b8d172ad3dcee"},"cell_type":"markdown","source":"I had to split my test set into multiple files for easier computation and running on kaggle kernels. If you have sufficient RAM  no need to go through these tribulations."},{"metadata":{"trusted":true,"_uuid":"1744237f1696ca45a697fb16b02515bdd49340dd"},"cell_type":"code","source":"for i_c,fn in enumerate(test_files1):\n    full_test1 = pickle.load(gzip.open(fn, 'rb'))\n    test2fn = '../input/create-test-set-new-feat/'+ fn.split('/')[-1]\n    test3fn = '../input/create-test-set-sionkowski-feat/'+ fn.split('/')[-1]\n    test4fn = '../input/create-test-set-detected-feat/'+ fn.split('/')[-1]\n#     print(test2fn)\n    full_test2 = pickle.load(gzip.open(test2fn, 'rb'))\n    full_test3 = pickle.load(gzip.open(test3fn, 'rb'))\n    full_test4 = pickle.load(gzip.open(test4fn, 'rb'))\n    \n    full_test = full_test1.merge(right=full_test2,how='outer',on='object_id')\n    full_test = full_test.merge(right=full_test3,how='outer',on='object_id')\n    full_test = full_test.merge(right=full_test4,how='outer',on='object_id')\n\n    object_ids = full_test.object_id.values\n    full_test = full_test[full_train_new.columns]\n    full_test_ss = ss.transform(np.nan_to_num(full_test))\n    \n#     Make predictions\n    preds = None\n    for clf in clfs:\n        if preds is None:\n            preds = clf.predict(full_test_ss,batch_size=batch_size) / folds.n_splits\n        else:\n            preds += clf.predict(full_test_ss,batch_size=batch_size) / folds.n_splits\n    if i_c % 10 == 0:\n        print(i_c+1,'files done')\n   # Compute preds_99 as the proba of class not being any of the others\n    # preds_99 = 0.1 gives 1.769\n    preds_99 = np.ones(preds.shape[0])\n    for i in range(preds.shape[1]):\n        preds_99 *= (1 - preds[:, i])\n\n    # Store predictions\n    preds_df = pd.DataFrame(preds, columns=class_names)\n    preds_df['object_id'] = object_ids\n    preds_df['class_99'] = 0.18 * preds_99 / np.mean(preds_99) \n    \n    if i_c == 0:\n        preds_df.to_csv('predictions.csv',  header=True, mode='a', index=False)\n    else: \n        preds_df.to_csv('predictions.csv',  header=False, mode='a', index=False)\n        \n    del full_test, preds_df, preds\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab549b5d9fc196d47d2a5d7b73f241dda07cdd8c"},"cell_type":"code","source":"z = pd.read_csv('predictions.csv')\n\nprint(z.groupby('object_id').size().max())\nprint((z.groupby('object_id').size() > 1).sum())\n\nz = z.groupby('object_id').mean()\n\n# z.to_csv('single_predictions.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f32157f08ddf43a379169e575f82272a273597d"},"cell_type":"markdown","source":"\n# Probability Correction"},{"metadata":{"_uuid":"b350ee2bc0c051d81ce8c67d588053050df9280d"},"cell_type":"markdown","source":"1. For objects with hostgal_specz=0\n\n Ideally mean for each extra-galactic class should be exactly 0 because according to hostgal_specz the classes can only be galactic classes\n\n2. For objects with hostgal_specz!=0\n\n Ideally mean for each galactic class should be exactly 0 because according to hostgal_specz the classes can only be extra galactic classes\n\n"},{"metadata":{"trusted":true,"_uuid":"2ef59de1a0101ee90b0c5b9516d03772a0275a6f"},"cell_type":"code","source":"test_meta_data = pd.read_csv('../input/PLAsTiCC-2018/test_set_metadata.csv')\ngal_obj = test_meta_data[test_meta_data.hostgal_photoz==0].object_id\nex_gal_obj = test_meta_data[test_meta_data.hostgal_photoz!=0].object_id\n\nprint('Percentage of galactic object in test set based on hostgal_photoz',len(gal_obj)/len(test_meta_data))\nprint('Percentage of extra galactic object in test set based on hostgal_photoz',\n      len(ex_gal_obj)/len(test_meta_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80c619d9c3122adfdbe0e24dd19e811e30d0c5aa"},"cell_type":"code","source":"gal_classes = [ 6, 16, 53, 65, 92]\ngal_cls_name = []\nfor val in gal_classes:\n    gal_cls_name.append('class_' + str(val))\n    \nex_gal_classes = [15, 42, 52, 62, 64, 67, 88, 90, 95]\nex_gal_cls_name = []\nfor val in ex_gal_classes:\n    ex_gal_cls_name.append('class_' + str(val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2997172584691bd6884ba42fe02de1cd7106bbe6"},"cell_type":"code","source":"final_sub = z.copy()\nfinal_sub.loc[gal_obj,gal_cls_name].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1ab63c93a0d874bb01dca21df759386c3ea1ac3"},"cell_type":"code","source":"final_sub.loc[gal_obj,ex_gal_cls_name] = 0 \nfinal_sub.loc[ex_gal_obj,gal_cls_name] = 0 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6645cecb59e36b01af3838af72920d805b070644"},"cell_type":"code","source":"final_sub.corrwith(z)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd23bf550eb245ea1e6a5b45b4acb6cada481d62"},"cell_type":"code","source":"final_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8931061f34123e1943236929378ab432cfaec585"},"cell_type":"code","source":"final_sub.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b5216d4bce72985e93b21a2f74620dda831aa43"},"cell_type":"code","source":"final_sub.to_csv('nn_sub.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}