# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load in import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the "../input/" directory. # For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory import os print(os.listdir("../input")) # Any results you write to the current directory are saved as output. # Loading Libraries import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) from sklearn.metrics import log_loss from sklearn.model_selection import StratifiedKFold import gc import os import matplotlib.pyplot as plt import seaborn as sns import lightgbm as lgb from catboost import Pool, CatBoostClassifier import itertools import pickle, gzip import glob from sklearn.preprocessing import StandardScaler from keras.models import Sequential from keras.layers import Dense,BatchNormalization,Dropout from keras.callbacks import ReduceLROnPlateau,ModelCheckpoint from keras.utils import to_categorical import tensorflow as tf from keras import backend as K import keras from keras import regularizers from collections import Counter from sklearn.metrics import confusion_matrix # Extracting Features from train set gc.enable() train = pd.read_csv('../input/training_set.csv') train['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0) train['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq'] aggs = { 'mjd': ['min', 'max', 'size'], 'passband': ['min', 'max', 'mean', 'median', 'std'], 'flux': ['min', 'max', 'mean', 'median', 'std','skew'], 'flux_err': ['min', 'max', 'mean', 'median', 'std','skew'], 'detected': ['mean'], 'flux_ratio_sq':['sum','skew'], 'flux_by_flux_ratio_sq':['sum','skew'], } agg_train = train.groupby('object_id').agg(aggs) new_columns = [ k + '_' + agg for k in aggs.keys() for agg in aggs[k] ] agg_train.columns = new_columns agg_train['mjd_diff'] = agg_train['mjd_max'] - agg_train['mjd_min'] agg_train['flux_diff'] = agg_train['flux_max'] - agg_train['flux_min'] agg_train['flux_dif2'] = (agg_train['flux_max'] - agg_train['flux_min']) / agg_train['flux_mean'] agg_train['flux_w_mean'] = agg_train['flux_by_flux_ratio_sq_sum'] / agg_train['flux_ratio_sq_sum'] agg_train['flux_dif3'] = (agg_train['flux_max'] - agg_train['flux_min']) / agg_train['flux_w_mean'] del agg_train['mjd_max'], agg_train['mjd_min'] agg_train.head() del train gc.collect() # Merging extracted features with meta data meta_train = pd.read_csv('../input/training_set_metadata.csv') meta_train.head() full_train = agg_train.reset_index().merge( right=meta_train, how='outer', on='object_id' ) if 'target' in full_train: y = full_train['target'] del full_train['target'] classes = sorted(y.unique()) # Taken from Giba's topic : https://www.kaggle.com/titericz # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194 # with Kyle Boone's post https://www.kaggle.com/kyleboone class_weight = { c: 1 for c in classes } for c in [64, 15]: class_weight[c] = 2 print('Unique classes : ', classes) if 'object_id' in full_train: oof_df = full_train[['object_id']] del full_train['object_id'], full_train['distmod'], full_train['hostgal_specz'] del full_train['ra'], full_train['decl'], full_train['gal_l'],full_train['gal_b'],full_train['ddf'] train_mean = full_train.mean(axis=0) full_train.fillna(train_mean, inplace=True) folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1) # Standard Scalling the input full_train_new = full_train.copy() ss = StandardScaler() full_train_ss = ss.fit_transform(full_train_new) def mywloss(y_true,y_pred): yc=tf.clip_by_value(y_pred,1e-15,1-1e-15) loss=-(tf.reduce_mean(tf.reduce_mean(y_true*tf.log(yc),axis=0)/wtable)) return loss def multi_weighted_logloss(y_ohe, y_p): classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95] class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1} # Normalize rows and limit y_preds to 1e-15, 1-1e-15 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15) # Transform to log y_p_log = np.log(y_p) # Get the log for ones, .values is used to drop the index of DataFrames # Exclude class 99 for now, since there is no class99 in the training set # we gave a special process for that class y_log_ones = np.sum(y_ohe * y_p_log, axis=0) # Get the number of positives for each class nb_pos = y_ohe.sum(axis=0).astype(float) # Weight average and divide by the number of positives class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())]) y_w = y_log_ones * class_arr / nb_pos loss = - np.sum(y_w) / np.sum(class_arr) return loss # Defining simple model in keras K.clear_session() def build_model(dropout_rate=0.25,activation='relu'): start_neurons = 512 # create model model = Sequential() model.add(Dense(start_neurons, input_dim=full_train_ss.shape[1], activation=activation)) model.add(BatchNormalization()) model.add(Dropout(dropout_rate)) model.add(Dense(start_neurons//2,activation=activation)) model.add(BatchNormalization()) model.add(Dropout(dropout_rate)) model.add(Dense(start_neurons//4,activation=activation)) model.add(BatchNormalization()) model.add(Dropout(dropout_rate)) model.add(Dense(start_neurons//8,activation=activation)) model.add(BatchNormalization()) model.add(Dropout(dropout_rate/2)) model.add(Dense(len(classes), activation='softmax')) return model unique_y = np.unique(y) class_map = dict() for i,val in enumerate(unique_y): class_map[val] = i y_map = np.zeros((y.shape[0],)) y_map = np.array([class_map[val] for val in y]) y_categorical = to_categorical(y_map) # Calculating the class weights y_count = Counter(y_map) wtable = np.zeros((len(unique_y),)) for i in range(len(unique_y)): wtable[i] = y_count[i]/y_map.shape[0] def plot_loss_acc(history): plt.plot(history.history['loss'][1:]) plt.plot(history.history['val_loss'][1:]) plt.title('model loss') plt.ylabel('val_loss') plt.xlabel('epoch') plt.legend(['train','Validation'], loc='upper left') plt.show() plt.plot(history.history['acc'][1:]) plt.plot(history.history['val_acc'][1:]) plt.title('model Accuracy') plt.ylabel('val_acc') plt.xlabel('epoch') plt.legend(['train','Validation'], loc='upper left') plt.show() clfs = [] oof_preds = np.zeros((len(full_train_ss), len(classes))) epochs = 600 batch_size = 100 for fold_, (trn_, val_) in enumerate(folds.split(y_map, y_map)): checkPoint = ModelCheckpoint("./keras.model",monitor='val_loss',mode = 'min', save_best_only=True, verbose=0) x_train, y_train = full_train_ss[trn_], y_categorical[trn_] x_valid, y_valid = full_train_ss[val_], y_categorical[val_] model = build_model(dropout_rate=0.5,activation='tanh') model.compile(loss=mywloss, optimizer='adam', metrics=['accuracy']) history = model.fit(x_train, y_train, validation_data=[x_valid, y_valid], epochs=epochs, batch_size=batch_size,shuffle=True,verbose=0,callbacks=[checkPoint]) plot_loss_acc(history) print('Loading Best Model') model.load_weights('./keras.model') # # Get predicted probabilities for each class oof_preds[val_, :] = model.predict_proba(x_valid,batch_size=batch_size) print(multi_weighted_logloss(y_valid, model.predict_proba(x_valid,batch_size=batch_size))) clfs.append(model) print('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_categorical,oof_preds)) # http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') print(cm) plt.imshow(cm, interpolation='nearest', cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) fmt = '.2f' if normalize else 'd' thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.ylabel('True label') plt.xlabel('Predicted label') plt.tight_layout() # Compute confusion matrix cnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1)) np.set_printoptions(precision=2) sample_sub = pd.read_csv('../input/sample_submission.csv') class_names = list(sample_sub.columns[1:-1]) del sample_sub;gc.collect() # Plot non-normalized confusion matrix plt.figure(figsize=(12,12)) foo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True, title='Confusion matrix') # Test Set prediction meta_test = pd.read_csv('../input/test_set_metadata.csv') import time start = time.time() chunks = 5000000 for i_c, df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)): df['flux_ratio_sq'] = np.power(df['flux'] / df['flux_err'], 2.0) df['flux_by_flux_ratio_sq'] = df['flux'] * df['flux_ratio_sq'] # Group by object id agg_test = df.groupby('object_id').agg(aggs) agg_test.columns = new_columns agg_test['mjd_diff'] = agg_test['mjd_max'] - agg_test['mjd_min'] agg_test['flux_diff'] = agg_test['flux_max'] - agg_test['flux_min'] agg_test['flux_dif2'] = (agg_test['flux_max'] - agg_test['flux_min']) / agg_test['flux_mean'] agg_test['flux_w_mean'] = agg_test['flux_by_flux_ratio_sq_sum'] / agg_test['flux_ratio_sq_sum'] agg_test['flux_dif3'] = (agg_test['flux_max'] - agg_test['flux_min']) / agg_test['flux_w_mean'] del agg_test['mjd_max'], agg_test['mjd_min'] # del df # gc.collect() # Merge with meta data full_test = agg_test.reset_index().merge( right=meta_test, how='left', on='object_id' ) full_test[full_train.columns] = full_test[full_train.columns].fillna(train_mean) full_test_ss = ss.transform(full_test[full_train.columns]) # Make predictions preds = None for clf in clfs: if preds is None: preds = clf.predict_proba(full_test_ss) / folds.n_splits else: preds += clf.predict_proba(full_test_ss) / folds.n_splits # Compute preds_99 as the proba of class not being any of the others # preds_99 = 0.1 gives 1.769 preds_99 = np.ones(preds.shape[0]) for i in range(preds.shape[1]): preds_99 *= (1 - preds[:, i]) # Store predictions preds_df = pd.DataFrame(preds, columns=class_names) preds_df['object_id'] = full_test['object_id'] preds_df['class_99'] = 0.14 * preds_99 / np.mean(preds_99) if i_c == 0: preds_df.to_csv('predictions.csv', header=True, mode='a', index=False) else: preds_df.to_csv('predictions.csv', header=False, mode='a', index=False) del agg_test, full_test, preds_df, preds # print('done') if (i_c + 1) % 10 == 0: print('%15d done in %5.1f' % (chunks * (i_c + 1), (time.time() - start) / 60)) z = pd.read_csv('predictions.csv') print(z.groupby('object_id').size().max()) print((z.groupby('object_id').size() > 1).sum()) z = z.groupby('object_id').mean() z.to_csv('single_predictions.csv', index=True) z.head()