{"cells":[{"metadata":{"_uuid":"1c60e7b501349e26f4aeaf4394bbf6a6a4693ed1"},"cell_type":"markdown","source":"* This is a starter script to get you started with pipelines\n* It is nowhere near optimized and doesn't reward the download and submit crowd so you will have to do some work for a reward!\n* The pipeline tries to classify into galaxy or intergalactic and then tries to predict the actual classes "},{"metadata":{"trusted":true,"_uuid":"3611b9bc84469ac7315032fe13f133d5cf3c4b2b"},"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.metrics import log_loss\nfrom scipy.stats import skew, kurtosis\n\nfrom sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29ea5e7b9eaea04f9d23e172885dfe72c6fb0c44"},"cell_type":"code","source":"def get_inputs(data, metadata):\n    data['flux_ratio_sq'] = np.power(data['flux'] / data['flux_err'], 2.0)\n    data['flux_by_flux_ratio_sq'] = data['flux'] * data['flux_ratio_sq']\n    mjddelta = train[['object_id','detected','mjd']]\n    mjddelta = mjddelta[mjddelta.detected==1].groupby('object_id').agg({'mjd': ['min', 'max']})\n    mjddelta['delta'] = mjddelta[mjddelta.columns[1]]-mjddelta[mjddelta.columns[0]]\n    mjddelta = mjddelta['delta'].reset_index(drop=False)\n    metadata = metadata.merge(mjddelta,on='object_id')\n    \n    \n    aggdata = data.groupby(['object_id','passband']).agg({'mjd': ['min', 'max', 'size'],\n                                             'flux': ['min', 'max', 'mean', 'median', 'std','skew'],\n                                             'flux_err': ['min', 'max', 'mean', 'median', 'std','skew'],\n                                             'flux_by_flux_ratio_sq': ['sum'],    \n                                             'flux_ratio_sq': ['sum'],                      \n                                             'detected': ['mean','std']}).reset_index(drop=False)\n    \n    cols = ['_'.join(str(s).strip() for s in col if s) if len(col)==2 else col for col in aggdata.columns ]\n    aggdata.columns = cols\n    aggdata = aggdata.merge(metadata,on='object_id',how='left')\n    aggdata.insert(1,'delta_passband', aggdata.mjd_max-aggdata.mjd_min)\n    aggdata.drop(['mjd_min','mjd_max'],inplace=True,axis=1)\n    aggdata['flux_diff'] = aggdata['flux_max'] - aggdata['flux_min']\n    aggdata['flux_dif2'] = (aggdata['flux_max'] - aggdata['flux_min']) / aggdata['flux_mean']\n    aggdata['flux_w_mean'] = aggdata['flux_by_flux_ratio_sq_sum'] / aggdata['flux_ratio_sq_sum']\n    aggdata['flux_dif3'] = (aggdata['flux_max'] - aggdata['flux_min']) / aggdata['flux_w_mean']\n    return aggdata\n\nclass GalaxyClassifier(BaseEstimator, TransformerMixin):\n    def __init__(self, lgb_params=None, cv=5):\n        self.lgb_params = lgb_params\n        self.cv = cv\n        self.clfs = []\n    \n    def fit(self, X, y=None):\n        importances = pd.DataFrame()\n        self.features = list(set(X.columns).difference(set(['object_id','hostgal_specz'])))\n        target = (X.hostgal_specz==0).astype(int)\n        folds = KFold(n_splits=self.cv, shuffle=True, random_state=1)\n        \n        oof_preds = np.zeros((len(X)))\n        for fold_, (trn_, val_) in enumerate(folds.split(target)):\n            trn_x, trn_y = X.iloc[trn_][self.features], target.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], target.iloc[val_]\n\n            clf = lgb.LGBMClassifier(**self.lgb_params)\n            clf.fit(\n                trn_x, trn_y,\n                eval_set=[(trn_x, trn_y), (val_x, val_y)],\n                eval_metric='binary_logloss',\n                verbose=1000,\n                early_stopping_rounds=50\n            )\n            imp_df = pd.DataFrame()\n            imp_df['feature'] = self.features\n            imp_df['gain'] = clf.feature_importances_\n            imp_df['fold'] = fold_ + 1\n            importances = pd.concat([importances, imp_df], sort=False)\n            preds = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:,1]\n            oof_preds[val_] = preds\n            print(log_loss(val_y, clf.predict_proba(val_x, num_iteration=clf.best_iteration_)))\n            self.clfs.append(clf)\n        mean_gain = importances[['gain', 'feature']].groupby('feature').mean()\n        importances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n        plt.figure(figsize=(8, 12))\n        sns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))\n        plt.tight_layout()\n        plt.show()\n        return self\n    \n    def fit_transform(self, X, y):\n        self.fit(X,y)\n        target = (X.hostgal_specz==0).astype(int)\n        oof_preds = np.zeros((len(X)))\n                \n        target = (X.hostgal_specz==0).astype(int)\n        folds = KFold(n_splits=self.cv, shuffle=True, random_state=1)\n        for fold_, (trn_, val_) in enumerate(folds.split(target)):\n            trn_x, trn_y = X.iloc[trn_][self.features], target.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], target.iloc[val_]\n            clf = self.clfs[fold_]\n            preds = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:,1]\n            oof_preds[val_] = preds\n        \n        X['hostgal_specz_pred'] = oof_preds\n        mnx = X[['object_id','hostgal_specz_pred']].groupby('object_id').hostgal_specz_pred.mean().reset_index(drop=False)\n        mnx.columns = ['object_id','hostgal_specz_pred']\n        X.drop('hostgal_specz_pred',inplace=True,axis=1)\n        X = X.merge(mnx,on='object_id',how='left')\n        X.loc[X.hostgal_specz==0,'hostgal_specz_pred'] = 1.0\n        return X\n        \n    def transform(self, X):\n        target = (X.hostgal_specz==0).astype(int)\n        oof_preds = np.zeros((len(X)))\n        for clf in self.clfs:\n            oof_preds += (clf.predict_proba(X[self.features], num_iteration=clf.best_iteration_)[:,1])/self.cv\n        \n        X['hostgal_specz_pred'] = oof_preds\n        mnx = X[['object_id','hostgal_specz_pred']].groupby('object_id').hostgal_specz_pred.mean().reset_index(drop=False)\n        mnx.columns = ['object_id','hostgal_specz_pred']\n        X.drop('hostgal_specz_pred',inplace=True,axis=1)\n        X = X.merge(mnx,on='object_id',how='left')\n        X.loc[X.hostgal_specz==0,'hostgal_specz_pred'] = 1.0\n        return X\n\nclass RealClassifier(BaseEstimator, ClassifierMixin):\n    def __init__(self, lgb_params=None, cv=5):\n        self.lgb_params = lgb_params\n        self.cv = cv\n        self.galaxyclfs = []\n        self.intergalacticclfs = []\n        self.standard_classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n        self.standard_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n        \n    def lgb_multi_weighted_logloss(self,y_true, y_preds):\n        \"\"\"\n        @author olivier https://www.kaggle.com/ogrellier\n        multi logloss for PLAsTiCC challenge\n        \"\"\"\n        # class_weights taken from Giba's topic : https://www.kaggle.com/titericz\n        # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n        # with Kyle Boone's post https://www.kaggle.com/kyleboone\n        classes = list(np.unique(y_true))\n        classes = [self.standard_classes[x] for x in classes]\n        class_weight = {i:self.standard_weight[i] for i in classes if self.standard_weight[i] is not None}\n\n\n        if len(np.unique(y_true)) > 14:\n            classes.append(99)\n            class_weight[99] = 2\n        y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n\n        # Trasform y_true in dummies\n        y_ohe = pd.get_dummies(y_true)\n        # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n        y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n        # Transform to log\n        y_p_log = np.log(y_p)\n        # Get the log for ones, .values is used to drop the index of DataFrames\n        # Exclude class 99 for now, since there is no class99 in the training set\n        # we gave a special process for that class\n        y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n        # Get the number of positives for each class\n        nb_pos = y_ohe.sum(axis=0).values.astype(float)\n        # Weight average and divide by the number of positives\n        class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n        y_w = y_log_ones * class_arr / nb_pos\n\n        loss = - np.sum(y_w) / np.sum(class_arr)\n        return 'wloss', loss, False\n    \n    def fitGalactic(self, X, y):\n        importances = pd.DataFrame()\n        folds = StratifiedKFold(n_splits=self.cv, shuffle=True, random_state=1)\n\n        X = X.loc[X.hostgal_specz==0].copy()\n        y = y.iloc[X.index].copy()\n        \n        oof_preds = np.zeros((len(X), np.unique(y).shape[0]))\n        lgb_params = self.lgb_params \n        lgb_params.update({'num_class': len(y.unique())})\n        for fold_, (trn_, val_) in enumerate(folds.split(X.index, y)):\n            trn_x, trn_y = X.iloc[trn_][self.features], y.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], y.iloc[val_]\n\n            clf = lgb.LGBMClassifier(**lgb_params)\n            clf.fit(\n                trn_x, trn_y,\n                eval_set=[(trn_x, trn_y), (val_x, val_y)],\n                eval_metric=self.lgb_multi_weighted_logloss,\n                verbose=1000,\n                early_stopping_rounds=50\n            )\n            imp_df = pd.DataFrame()\n            imp_df['feature'] = self.features\n            imp_df['gain'] = clf.feature_importances_\n            imp_df['fold'] = fold_ + 1\n            importances = pd.concat([importances, imp_df], sort=False)\n            self.galaxyclfs.append(clf)\n        mean_gain = importances[['gain', 'feature']].groupby('feature').mean()\n        importances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n        plt.figure(figsize=(8, 12))\n        sns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))\n        plt.tight_layout()\n        plt.show()\n        return self\n    \n    def fitInterGalactic(self, X, y):\n        importances = pd.DataFrame()\n        folds = StratifiedKFold(n_splits=self.cv, shuffle=True, random_state=1)\n       \n        X = X.loc[X.hostgal_specz!=0].copy()\n        y = y.iloc[X.index].copy()\n        \n        \n        \n        oof_preds = np.zeros((len(X), np.unique(y).shape[0]))\n        lgb_params = self.lgb_params \n        lgb_params.update({'num_class': len(y.unique())})\n        for fold_, (trn_, val_) in enumerate(folds.split(X.index, y)):\n            trn_x, trn_y = X.iloc[trn_][self.features], y.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], y.iloc[val_]\n\n            clf = lgb.LGBMClassifier(**lgb_params)\n            clf.fit(\n                trn_x, trn_y,\n                eval_set=[(trn_x, trn_y), (val_x, val_y)],\n                eval_metric=self.lgb_multi_weighted_logloss,\n                verbose=1000,\n                early_stopping_rounds=50\n            )\n            imp_df = pd.DataFrame()\n            imp_df['feature'] = self.features\n            imp_df['gain'] = clf.feature_importances_\n            imp_df['fold'] = fold_ + 1\n            importances = pd.concat([importances, imp_df], sort=False)\n            self.intergalacticclfs.append(clf)\n        mean_gain = importances[['gain', 'feature']].groupby('feature').mean()\n        importances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n        plt.figure(figsize=(8, 12))\n        sns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))\n        plt.tight_layout()\n        plt.show()\n        return self\n\n      \n    def fit(self, X, y):\n        X = X.copy()\n        self.features = list(set(X.columns).difference(set(['object_id','target','hostgal_specz'])))\n        self.fitGalactic(X, y)   \n        self.fitInterGalactic(X, y)   \n        return self\n    \n    \n    def fit_transform(self, X, y):\n        self.fit(X,y)\n        oof_preds = np.zeros((len(X),len(self.galaxyclfs[0].classes_)))\n        \n        folds = StratifiedKFold(n_splits=self.cv, shuffle=True, random_state=1)\n        for fold_, (trn_, val_) in enumerate(folds.split(X.index, y)):\n            trn_x, trn_y = X.iloc[trn_][self.features], y.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], y.iloc[val_]\n            clf = self.galaxyclfs[fold_]\n            preds = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n            oof_preds[val_] = preds\n        \n        oof_preds1 = np.zeros((len(X),len(self.intergalacticclfs[0].classes_)))\n        folds = StratifiedKFold(n_splits=self.cv, shuffle=True, random_state=1)\n        for fold_, (trn_, val_) in enumerate(folds.split(X.index, y)):\n            trn_x, trn_y = X.iloc[trn_][self.features], y.iloc[trn_]\n            val_x, val_y = X.iloc[val_][self.features], y.iloc[val_]\n            clf = self.intergalacticclfs[fold_]\n            preds = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n            oof_preds1[val_] = preds\n        \n        galaxy = pd.DataFrame(oof_preds)\n        galaxy.columns = ['class_'+str(i) for i in self.galaxyclfs[0].classes_]\n        intergalactic = pd.DataFrame(oof_preds1)\n        intergalactic.columns = ['class_'+str(i) for i in self.intergalacticclfs[0].classes_]\n        alltargets = list(set(list(galaxy.columns)+list(intergalactic.columns)))\n        x = pd.DataFrame()\n        x['object_id'] = X.object_id.ravel()\n        for c in alltargets:\n            if(c in galaxy):\n                x[c] = galaxy[c].values*(X['hostgal_specz_pred'])\n            if(c in intergalactic):\n                x[c] = intergalactic[c].values*(1-X['hostgal_specz_pred'].values)\n        x['class_99'] = (1- x[x.columns[1:]].sum(axis=1)).clip(0,1)\n        x = x.groupby(['object_id']).mean().reset_index(drop=False)\n        return x.fillna(0)\n    \n    \n    def transform(self, X):\n        X = X.copy()\n        oof_preds = np.zeros((len(X),len(self.galaxyclfs[0].classes_)))\n        \n        for clf in self.galaxyclfs:\n            oof_preds += (clf.predict_proba(X[self.features], num_iteration=clf.best_iteration_))/self.cv\n        \n        oof_preds1 = np.zeros((len(X),len(self.intergalacticclfs[0].classes_)))\n        \n        for clf in self.intergalacticclfs:\n            oof_preds1 += (clf.predict_proba(X[self.features], num_iteration=clf.best_iteration_))/self.cv\n               \n        galaxy = pd.DataFrame(oof_preds)\n        galaxy.columns = ['class_'+str(i) for i in self.galaxyclfs[0].classes_]\n        intergalactic = pd.DataFrame(oof_preds1)\n        intergalactic.columns = ['class_'+str(i) for i in self.intergalacticclfs[0].classes_]\n        alltargets = list(set(list(galaxy.columns)+list(intergalactic.columns)))\n        x = pd.DataFrame()\n        x['object_id'] = X.object_id.ravel()\n        for c in alltargets:\n            if(c in galaxy):\n                x[c] = galaxy[c].values*(X['hostgal_specz_pred'])\n            if(c in intergalactic):\n                x[c] = intergalactic[c].values*(1-X['hostgal_specz_pred'].values)\n        x['class_99'] = (1- x[x.columns[1:]].sum(axis=1)).clip(0,1)\n        x = x.groupby(['object_id']).mean().reset_index(drop=False)\n        return x.fillna(0)\n    \ndef multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    # class_weights taken from Giba's topic : https://www.kaggle.com/titericz\n    # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n    # with Kyle Boone's post https://www.kaggle.com/kyleboone\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8844d754f0b1039d0d59a73628717685df2df4a5"},"cell_type":"code","source":"train = pd.read_csv('../input/training_set.csv')\nmeta_train = pd.read_csv('../input/training_set_metadata.csv')\ntraindata = get_inputs(train,meta_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddb1caa2131d649e38f488a2a5cfed73f317ac8a"},"cell_type":"code","source":"traindata.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f1728546e0b1cb8ea160ab7192bcaf644032ab7"},"cell_type":"code","source":"cols = list(set(traindata.columns).difference(set(['target'])))\nprint(cols)\nX = traindata[cols].copy()\ny = traindata.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cf18b1a846a765f04557447890460c4d56c3ddb"},"cell_type":"code","source":"lgb_params = {\n            'objective': 'binary',\n            'boosting': 'gbdt',\n            'learning_rate': 0.1 ,\n            'silent': 1,\n            'verbose': 0,\n            'subsample': .9,\n            'colsample_bytree': .7,\n            'metric' : 'binary_logloss',\n            'max_depth': 3,\n            'min_child_weight':10\n        }\n\nal_lgb_params = {\n            'objective': 'multiclass',\n            'metric': 'multi_logloss',\n            'learning_rate': 0.1,\n            'subsample': .9,\n            'colsample_bytree': .7,\n            'reg_lambda': 1.0,\n            'min_split_gain': 0.01,\n            'n_estimators': 1000,\n            'silent': 1,\n            'verbose': 0,\n            'max_depth': 3,\n            'min_child_weight':10}\n\n\npipe = Pipeline([('gl', GalaxyClassifier(lgb_params=lgb_params, cv=5)),\n                 ('al', RealClassifier(lgb_params=al_lgb_params, cv=5))])\npredictions = pipe.fit_transform(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2fb346570cc4bae243736cf75a723be96ad9a39"},"cell_type":"code","source":"predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"733c34837f6d5fee394ba67acd4204309601e8d1"},"cell_type":"code","source":"cols = pd.get_dummies(meta_train.target).columns.values\ncols = ['class_'+str(c) for c in cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"799279b7dd9e004d70dd7c7b05f5042fcd733f4c"},"cell_type":"code","source":"log_loss(pd.get_dummies(meta_train.target), predictions[cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"325cd61d9f97d60b60a69ad3644864ec7f6655f7"},"cell_type":"code","source":"multi_weighted_logloss(meta_train.target,predictions[cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50e485259e276ba3a8fab24b1217613ab5699008"},"cell_type":"code","source":"model = TSNE(n_components=2, perplexity=30,random_state=0)\ntsnedata = model.fit_transform(predictions[predictions.columns[1:]])\ncm = plt.cm.get_cmap('RdYlBu')\nfig, axes = plt.subplots(1, 1, figsize=(15, 15))\nsc = axes.scatter(tsnedata[:,0], tsnedata[:,1], alpha=1, c=(meta_train.target.values), cmap=cm, s=1)\ncbar = fig.colorbar(sc, ax=axes)\ncbar.set_label('Target')\n_ = axes.set_title(\"Clustering colored by target\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d176d3fcd60a97a67805a4ffa16fe3ae6028d111"},"cell_type":"code","source":"# meta_test = pd.read_csv('../input/test_set_metadata.csv')\n# print(meta_test.shape)\n# chunks = 5000000\n# remain_df = None\n# test_all_predictions = None\n# for i_c, df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)):\n#     print(i_c,df.shape)\n#     unique_ids = np.unique(df['object_id'])\n#     new_remain_df = df.loc[df['object_id'] == unique_ids[-1]].copy()\n#     if remain_df is None:\n#         df = df.loc[df['object_id'].isin(unique_ids[:-1])].copy()\n#     else:\n#         df = pd.concat([remain_df, df.loc[df['object_id'].isin(unique_ids[:-1])]], axis=0)\n#     # Create remaining samples df\n#     remain_df = new_remain_df\n\n#     df = get_inputs(df, meta_test)\n#     preds_df = pipe.transform(df)\n#     if i_c == 0:\n#         preds_df.to_csv('predictions.csv', header=True, index=False, float_format='%.6f')\n#     else:\n#         preds_df.to_csv('predictions.csv', header=False, mode='a', index=False, float_format='%.6f')\n\n#     del preds_df\n#     gc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}