{"cells":[{"metadata":{"_uuid":"609006646970a83c89ff8a332459ddceb2e02e4c"},"cell_type":"markdown","source":"# Loading Libraries"},{"metadata":{"trusted":true,"_uuid":"0d2859554c15163e5f5ed16a926244be8431bf71"},"cell_type":"code","source":"import os\nos.listdir('../input/lognormalizefeaturesetstogether')\nprint(os.listdir('../input/lognormalizefeaturesetstogetherwextrafeats'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport lightgbm as lgb\nfrom catboost import Pool, CatBoostClassifier\nimport itertools\nimport pickle, gzip\nimport glob\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7dbccfa992f29644a47a31341b2c68c6b42b835d"},"cell_type":"markdown","source":"# Extracting Features from train set"},{"metadata":{"trusted":true,"_uuid":"631c418da4275fba3070f2a6cff3873d11591f95"},"cell_type":"code","source":"traindf=pd.read_csv('../input/lognormalizefeaturesetstogetherwextrafeats/igTrain.csv')\n#del traindf['distmod']\n\nprint(traindf.shape)\ntraindf.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ed21e25fb5678c78bb19a8297bf83db00ccae01"},"cell_type":"markdown","source":"## Get y information"},{"metadata":{"trusted":true,"_uuid":"696ee870837c23375bc7f0388de1b07510351123"},"cell_type":"code","source":"\nfull_train = traindf\ndel traindf\nfull_train.shape\n\nif 'target' in full_train.columns:\n    y = full_train['target']\n    ydf=pd.DataFrame()\n    ydf['target']=full_train['target']\n    if 'object_id' in full_train.columns:\n        ydf['object_id']=full_train['object_id']\n    \n    del full_train['target']\nclasses = sorted(y.unique())\n\n# Taken from Giba's topic : https://www.kaggle.com/titericz\n# https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n# with Kyle Boone's post https://www.kaggle.com/kyleboone\nclass_weight = {\n    c: 1 for c in classes\n}\nfor c in [64, 15]:\n    class_weight[c] = 2\n\nprint('Unique classes : ', classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eefc49ef5164dd15065fd99d46fb9402240145f0"},"cell_type":"code","source":"ydf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7aaadd7fb68f26be1aa78e0721332689365f2322"},"cell_type":"markdown","source":"# Standard Scaling the input (imp.)"},{"metadata":{"trusted":true,"_uuid":"c1aaa4b97bec65047182097d08d33541aaf9a057"},"cell_type":"code","source":"#import copy\n#full_train_new = copy.deepcopy(full_train)\n#ss = StandardScaler()\n#full_train_ss = ss.fit_transform(full_train_new)\n\nif 'object_id' in full_train:\n    oof_df = full_train[['object_id']]\n    del full_train['object_id']\n\nfull_train_ss=full_train.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f641d2b24ea1164c8b1d2e57bed6c6633b7c125b"},"cell_type":"code","source":"\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.model_selection import train_test_split\nimport numpy as np # linear algebra\nimport pandas as pd\n\n#modify to work with kfold\n#def smoteAdataset(Xig, yig, test_size=0.2, random_state=0):\ndef smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):\n    \n        \n    sm=SMOTE(random_state=2)\n    Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())\n\n        \n    return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d019e33855e6e351f32e8e490851ac035c8a24b"},"cell_type":"code","source":"\n    \n#train_mean = full_train.mean(axis=0)\n#full_train.fillna(train_mean, inplace=True)\n\nfolds = StratifiedKFold(n_splits=12, shuffle=True, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92b0ae5a7f0b581b1f4e971a5c62eb97d2397455"},"cell_type":"markdown","source":"# Deep Learning Begins..."},{"metadata":{"trusted":true,"_uuid":"7b7424590dc985a79354838aa99ed5cb94ebc3f7"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense,BatchNormalization,Dropout\nfrom keras.callbacks import ReduceLROnPlateau,ModelCheckpoint\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nimport keras\nfrom keras import regularizers\nfrom collections import Counter\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59889ae9a9041a1c9efa8c42c59b75aa58249bf1"},"cell_type":"code","source":"# https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69795\ndef mywloss(y_true,y_pred):  \n    yc=tf.clip_by_value(y_pred,1e-15,1-1e-15)\n    loss=-(tf.reduce_mean(tf.reduce_mean(y_true*tf.log(yc),axis=0)/wtable))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c35b11a75cdc6aa926109ae876a1f2bb9f4078c"},"cell_type":"code","source":"def multi_weighted_logloss(y_ohe, y_p):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 16, 53, 65, 92]\n    class_weight = {6: 1, 16: 1, 53: 1, 65: 1, 92: 1}\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6601f669629d8e1bf9182054206ef316341e0e3e"},"cell_type":"markdown","source":"# Defining simple model in keras"},{"metadata":{"trusted":true,"_uuid":"4a411f6097378ab0c3199064478f4ed4e064b6fb"},"cell_type":"code","source":"K.clear_session()\ndef build_model(dropout_rate=0.35,activation='relu'):\n    start_neurons = 1024\n    # create model\n    model = Sequential()\n    model.add(Dense(start_neurons, input_dim=full_train_ss.shape[1], activation=activation))\n    model.add(BatchNormalization())\n    model.add(Dropout(dropout_rate))\n    \n    model.add(Dense(start_neurons//2,activation=activation))\n    model.add(BatchNormalization())\n    model.add(Dropout(dropout_rate))\n    \n    model.add(Dense(start_neurons//4,activation=activation))\n    model.add(BatchNormalization())\n    model.add(Dropout(dropout_rate))\n    \n    model.add(Dense(start_neurons//8,activation=activation))\n    model.add(BatchNormalization())\n    model.add(Dropout(dropout_rate/2))\n    \n    model.add(Dense(start_neurons//16,activation=activation))\n    model.add(BatchNormalization())\n    model.add(Dropout(dropout_rate/2))\n    \n    model.add(Dense(len(classes), activation='softmax'))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9235d7edc1520d10784f0f14b88e35651d7c9d5b"},"cell_type":"code","source":"unique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n        \ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\ny_categorical = to_categorical(y_map)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b915ef7ac26164d0b83e265a277ea50e8557eac"},"cell_type":"markdown","source":"# Calculating the class weights"},{"metadata":{"trusted":true,"_uuid":"fe95383168c5bd60e4349d6bce4999ffdff02471"},"cell_type":"code","source":"y_count = Counter(y_map)\nwtable = np.zeros((len(unique_y),))\nfor i in range(len(unique_y)):\n    wtable[i] = y_count[i]/y_map.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449a0c6c0f3baa75ef4719ef8e76ac5fa62d094b"},"cell_type":"code","source":"def plot_loss_acc(history):\n    plt.plot(history.history['loss'][1:])\n    plt.plot(history.history['val_loss'][1:])\n    plt.title('model loss')\n    plt.ylabel('val_loss')\n    plt.xlabel('epoch')\n    plt.legend(['train','Validation'], loc='upper left')\n    plt.show()\n    \n    plt.plot(history.history['acc'][1:])\n    plt.plot(history.history['val_acc'][1:])\n    plt.title('model Accuracy')\n    plt.ylabel('val_acc')\n    plt.xlabel('epoch')\n    plt.legend(['train','Validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6417136e31687df52d5d9f85a83df00aaa6cf2eb"},"cell_type":"code","source":"clfs = []\noof_preds = np.zeros((len(full_train_ss), len(classes)))\nepochs = 600\nbatch_size = 100\nfor fold_, (trn_, val_) in enumerate(folds.split(y_map, y_map)):\n    checkPoint = ModelCheckpoint(\"./keras.model\",monitor='val_loss',mode = 'min', save_best_only=True, verbose=0)\n    x_train, y_train = full_train_ss[trn_], y_categorical[trn_]\n    x_valid, y_valid = full_train_ss[val_], y_categorical[val_]\n    \n    #trn_xa, y_train, val_xa, y_valid=smoteAdataset(x_train, pd.Series(y_train), x_valid, pd.Series(y_valid))\n    #x_train=pd.DataFrame(data=trn_xa, columns=x_train.columns)\n\n    #x_valid=pd.DataFrame(data=val_xa, columns=x_valid.columns)\n\n    model = build_model(dropout_rate=0.5,activation='tanh')    \n    model.compile(loss=mywloss, optimizer='adam', metrics=['accuracy'])\n    history = model.fit(x_train, y_train,\n                    validation_data=[x_valid, y_valid], \n                    epochs=epochs,\n                    batch_size=batch_size,shuffle=True,verbose=0,callbacks=[checkPoint])       \n    \n    plot_loss_acc(history)\n    \n    print('Loading Best Model')\n    model.load_weights('./keras.model')\n    # # Get predicted probabilities for each class\n    oof_preds[val_, :] = model.predict_proba(x_valid,batch_size=batch_size)\n    print(multi_weighted_logloss(y_valid, model.predict_proba(x_valid,batch_size=batch_size)))\n    clfs.append(model)\n    \nscore=multi_weighted_logloss(y_categorical,oof_preds)\n\nprint('MULTI WEIGHTED LOG LOSS : %.5f ' % score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8d149b6795271bdce59ee091118bd1681cb3462"},"cell_type":"code","source":"# http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc5c23f2780db663e813e796b811f3831ef42686"},"cell_type":"code","source":"# Compute confusion matrix\n#cnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1))\n#np.set_printoptions(precision=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"161c3e5e83a2ee3927df2c782fcb53fb7a379292"},"cell_type":"code","source":"import pandas as pd\n#sample_sub = pd.read_csv('../input/PLAsTiCC-2018/sample_submission.csv')\n#class_names = list(sample_sub.columns[1:-1])\n#del sample_sub;\nclass_names=['class_6', 'class_16','class_53',\n            'class_65','class_92']\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9416f8a1b8b43e49f54642e03793ffce83adedd","scrolled":true},"cell_type":"code","source":"# Plot non-normalized confusion matrix\n#plt.figure(figsize=(12,12))\n#foo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True,\n#                      title='Confusion matrix')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f65492856178f1692de22989481d7dde8afd1e84"},"cell_type":"markdown","source":"# Test Set Predictions"},{"metadata":{"trusted":true,"_uuid":"4877f7bb6f625eec99ce6a59fcf532836c0267df"},"cell_type":"code","source":"testdf=pd.read_csv('../input/lognormalizefeaturesetstogetherwextrafeats/igTest.csv')\n#del testdf['distmod']\n\nif 'object_id' in testdf:\n    objidTest=pd.DataFrame()\n    objidTest.loc[:,'object_id']=testdf.loc[:,'object_id']\n    del testdf['object_id']\n\ntestdf.shape\n\ntestdf.describe()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d233a585e1e0fddc3ba73304c98cc68773ef961"},"cell_type":"code","source":"#df[df.columns] = scaler.fit_transform(df[df.columns])\n#ss = StandardScaler()\n#testdf[testdf.columns]=ss.fit_transform(testdf[testdf.columns])\n#testdf.describe()\n\ntestdf_ss=testdf.values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fc5fb61d3cd86a6efb1e309c84c5997d2e89628"},"cell_type":"code","source":"testdf['object_id']=objidTest['object_id']\ndel testdf['object_id']\ntestdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70be275d03dcd3ab0839f7c1215efb29ac939f37"},"cell_type":"code","source":"def predict_test(testdf, objidTest, clfs, features, class_names):\n    \n    preds = None\n    for clf in clfs:\n        if preds is None:\n            preds = clf.predict_proba(testdf[features]) / len(clfs)\n        else:\n            preds += clf.predict_proba(testdf[features]) / len(clfs)\n            \n    preds_99 = np.ones(preds.shape[0])\n    \n    \n    for i in range(preds.shape[1]):\n        preds_99 *= (1 - preds[:, i])\n\n    # Create DataFrame from predictions\n    preds_df = pd.DataFrame(preds, columns=class_names)\n    preds_df['object_id'] = objidTest['object_id']\n    preds_df['class_99'] = 0.14 * preds_99 / np.mean(preds_99)\n    \n    return preds_df\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecd1a90a6ee04cabb0da61ef892c127102d2ad6b"},"cell_type":"code","source":"pdf=predict_test(testdf, objidTest, clfs, full_train.columns, class_names)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e7dd7046c28e87d6acc690ed1b4c9afa31edf05"},"cell_type":"code","source":"pdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f54266624d7711d2fd015a43c7cea637b2977c25"},"cell_type":"code","source":"#from Scirpus discussion:\n\ndef GenUnknown(data):\n    return ((((((data[\"mymedian\"]) + (((data[\"mymean\"]) / 2.0)))/2.0)) + (((((1.0) - (((data[\"mymax\"]) * (((data[\"mymax\"]) * (data[\"mymax\"]))))))) / 2.0)))/2.0)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"362e4bbcf2a897294e5045a3a969f6fa8bc31066"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"272d0bfaef247f93e980e5086fee95a2af180006"},"cell_type":"code","source":"from datetime import datetime as dt\nfilename = 'subm_{:.6f}_{}.csv'.format(score, \n                 dt.now().strftime('%Y-%m-%d-%H-%M'))\nprint('save to {}'.format(filename))\n# TEST\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e4f02cae4ff7276c3c1532b0b66e58aef58d7d4"},"cell_type":"code","source":"# get a list of columns\ncols = list(pdf)\n# move the column to head of list using index, pop and insert\n#cols.insert(0, cols.pop(cols.index('object_id')))\n#pdf = pdf.loc[:, cols]\n\n\n\nfeats = ['class_6', 'class_16',  'class_53', 'class_65', 'class_92']\n\ny = pd.DataFrame()\ny['mymean'] = pdf[feats].mean(axis=1)\ny['mymedian'] = pdf[feats].median(axis=1)\ny['mymax'] = pdf[feats].max(axis=1)\n\npdf['class_99'] = GenUnknown(y)\npdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"705b3915095e941ff150d1307fe5c7cb768b0418"},"cell_type":"code","source":"pdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99166b5c98de7f41722d6319dd121d445828ba3"},"cell_type":"code","source":"pdf.to_csv(filename, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}