{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44245389c568e6bf6c045ecc5cc3ffad718ed0da"},"cell_type":"code","source":"def get_features_from_mjd(df):\n    df['mjd_detected'] = np.NaN\n    df.loc[df.detected == 1, 'mjd_detected'] = df.loc[df.detected == 1, 'mjd']\n    gr_mjd = df.groupby('object_id').mjd_detected\n    df['mjd_diff']  = gr_mjd.transform('max') - gr_mjd.transform('min')    \n    return df\n\ndef passbandSplit(df):\n    df[\"flux_0\"] = df[df.passband==0].flux\n    df[\"flux_1\"] = df[df.passband==1].flux\n    df[\"flux_2\"] = df[df.passband==2].flux\n    df[\"flux_3\"] = df[df.passband==3].flux\n    df[\"flux_4\"] = df[df.passband==4].flux\n    df[\"flux_5\"] = df[df.passband==5].flux\n    \n    df[\"abs_flux_1\"] = np.abs(df[\"flux_1\"])\n    df[\"abs_flux_2\"] = np.abs(df[\"flux_2\"])\n    df[\"abs_flux_3\"] = np.abs(df[\"flux_3\"])\n    df[\"abs_flux_4\"] = np.abs(df[\"flux_4\"])\n    df[\"abs_flux_5\"] = np.abs(df[\"flux_5\"])\n    \n    df[\"flux_0_err\"] = df[df.passband==0].flux_err\n    df[\"flux_1_err\"] = df[df.passband==1].flux_err\n    df[\"flux_4_err\"] = df[df.passband==4].flux_err\n    df[\"flux_5_err\"] = df[df.passband==5].flux_err\n    \n    df['flux_ratio_sq_0'] = np.power(df['flux_0'] / df['flux_0_err'], 2.0)\n    df['flux_ratio_sq_1'] = np.power(df['flux_1'] / df['flux_1_err'], 2.0)\n    df['flux_ratio_sq_4'] = np.power(df['flux_4'] / df['flux_4_err'], 2.0)\n    df['flux_ratio_sq_5'] = np.power(df['flux_5'] / df['flux_5_err'], 2.0)\n    \n    df['flux_by_flux_ratio_sq_0'] = df['flux_0'] * df['flux_ratio_sq_0']\n    df['flux_by_flux_ratio_sq_1'] = df['flux_1'] * df['flux_ratio_sq_1']\n    df['flux_by_flux_ratio_sq_5'] = df['flux_5'] * df['flux_ratio_sq_5']\n    return df\n\ndef sabun_henkaritsu_cumsum(df):\n    gr_df = df.groupby(['object_id','passband'])\n    \n    gr_flux = gr_df['flux']\n    df[\"flux_henkaritsu\"] = gr_flux.pct_change()\n\n    df[\"flux_sabun_diff\"] = gr_flux.transform('max') - gr_flux.transform('min') \n\n    gr_mag = gr_df['magnitude']\n    df[\"mag_sabun_diff\"] = gr_mag.transform('max') - gr_mag.transform('min') \n    df[\"mag_sabun\"] = gr_mag.diff()\n\n    gr_fl_ratio_sq = gr_df['flux_ratio_sq']\n    df[\"fl_ratio_sabun\"] = gr_fl_ratio_sq.diff()\n    df[\"flux_detected\"] = df[df.detected==1].flux\n    df[\"dtd_fl_by_mjd_dif\"] = df[\"flux_detected\"] / df['mjd_diff']\n    df[\"dtd_magnitude\"] = df[df.detected==1].magnitude\n    df[\"dtd_magnitude_1\"] = df[df.detected==1].magnitude_1\n    df[\"dtd_magnitude_2\"] = df[df.detected==1].magnitude_2\n    df[\"dtd_magnitude_3\"] = df[df.detected==1].magnitude_3\n\n    gr_ob = df.groupby('object_id')\n    df['dtdmag_diff']  = gr_ob.dtd_magnitude.transform('max') - gr_ob.dtd_magnitude.transform('min')\n    df['dtdmag_diff_by_mjd'] = df['dtdmag_diff'] / df['mjd_diff']\n    \n    df['dtdmag_1_diff']  = gr_ob.dtd_magnitude_1.transform('max') - gr_ob.dtd_magnitude_1.transform('min')\n    df['dtdmag_2_diff']  = gr_ob.dtd_magnitude_2.transform('max') - gr_ob.dtd_magnitude_2.transform('min')\n    df['dtdmag_3_diff']  = gr_ob.dtd_magnitude_3.transform('max') - gr_ob.dtd_magnitude_3.transform('min')\n    df['dtdmag_1_diff_by_mjd'] = df['dtdmag_1_diff'] / df['mjd_diff']\n    df['dtdmag_2_diff_by_mjd'] = df['dtdmag_2_diff'] / df['mjd_diff']\n    df['dtdmag_3_diff_by_mjd'] = df['dtdmag_3_diff'] / df['mjd_diff']\n    del df['dtdmag_1_diff'], df['dtdmag_2_diff'], df['dtdmag_3_diff']\n    df[\"dtdmag_pct\"] = gr_ob.dtd_magnitude.pct_change()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee6ebbf90212b528a98597116669b73d9229c242"},"cell_type":"code","source":"def get_flux_decays(df):\n    df['flux_1_err_diff_per_mjd'] = (df['flux_1_err_max'] - df['flux_1_err_min']) / df['mjd_diff_mean']    \n    del df['flux_1_err_max'], df['flux_1_err_min']\n\n    df[\"flux_5_4_max\"] = (df[\"flux_5_max\"] - df[\"flux_4_max\"])\n    df[\"flux_4_3_max\"] = (df[\"flux_4_max\"] - df[\"flux_3_max\"])\n    df[\"flux_3_2_max\"] = (df[\"flux_3_max\"] - df[\"flux_2_max\"])\n    df[\"flux_2_1_max\"] = (df[\"flux_2_max\"] - df[\"flux_1_max\"])\n    df[\"flux_1_0_max\"] = (df[\"flux_1_max\"] - df[\"flux_0_max\"])\n    df[\"flux_5_0_mean\"] = (df[\"flux_5_mean\"] - df[\"flux_0_mean\"])\n    df[\"flux_3_0_mean\"] = (df[\"flux_3_mean\"] - df[\"flux_0_mean\"])\n    df[\"flux_4_3_std\"] = (df[\"flux_4_std\"] - df[\"flux_3_std\"])\n    df[\"flux_3_2_std\"] = (df[\"flux_3_std\"] - df[\"flux_2_std\"])\n    df[\"flux_2_1_std\"] = (df[\"flux_2_std\"] - df[\"flux_1_std\"])\n    df[\"flux_5_0_median\"] = (df[\"flux_5_median\"] - df[\"flux_0_median\"])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1500ca2c0b07de9aa127bb01a304f7ba8d67f93f"},"cell_type":"code","source":"train_set = pd.read_csv('../input/training_set.csv', dtype={\"object_id\": \"object\"})\ntrain_set_meta = pd.read_csv('../input/training_set_metadata.csv', dtype={\"object_id\": \"object\"})\ntest_set_meta = pd.read_csv('../input/test_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff0d436cbffda3a6457b0254581636693d9e58c2"},"cell_type":"code","source":"import gc\ngc.enable()\n\ntrain_set = get_features_from_mjd(train_set)\ntrain_set = passbandSplit(train_set)\n\ntrain_set['flux_ratio_sq'] = np.power(train_set['flux'] / train_set['flux_err'], 2.0)\ntrain_set['flux_by_flux_ratio_sq'] = train_set['flux'] * train_set['flux_ratio_sq']\n\ntrain_set[\"magnitude\"] = -2.5 * np.log(train_set[\"flux\"]).fillna(0)\ntrain_set[\"magnitude_0\"] = -2.5 * np.log(train_set[\"flux_0\"]).fillna(0)\ntrain_set[\"magnitude_1\"] = -2.5 * np.log(train_set[\"flux_1\"]).fillna(0)\ntrain_set[\"magnitude_2\"] = -2.5 * np.log(train_set[\"flux_2\"]).fillna(0)\ntrain_set[\"magnitude_3\"] = -2.5 * np.log(train_set[\"flux_3\"]).fillna(0)\n\ntrain_set = sabun_henkaritsu_cumsum(train_set)\n\naggs = {\n    \"dtdmag_pct\": ['min', 'max', 'mean', 'median', 'std','skew', \"sum\"],\n    'dtdmag_1_diff_by_mjd': ['min'],\n    'dtdmag_2_diff_by_mjd': ['min'],\n    'dtdmag_3_diff_by_mjd': ['min'],\n    #\n    'dtdmag_diff': ['min', \"sum\"],\n    'dtdmag_diff_by_mjd': ['min', 'max', 'mean', 'median', \"sum\"],\n    \"dtd_magnitude\": ['max', 'mean', 'median', 'std','skew'],\n    \"dtd_magnitude_1\": ['mean', 'skew'],\n    \"dtd_magnitude_2\": ['mean', 'std','skew'],\n    \"dtd_magnitude_3\": ['std','skew'],\n    #\n    \"dtd_fl_by_mjd_dif\": ['max', 'std'],\n    \"flux_detected\": ['min', 'median', 'std','skew'],\n    #\n    'magnitude': ['min', 'max', 'mean', 'std','skew'],\n    'flux': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_err': ['min', 'std'],\n    'detected': ['mean'],\n    'flux_ratio_sq':['sum','skew'],\n    'flux_by_flux_ratio_sq':['sum','skew'],\n    'mjd_diff': ['mean'],\n    'mjd_detected': ['std'],\n\n    'flux_0': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_1': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_2': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_3': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_4': ['min', 'max', 'mean', 'median', 'std','skew'],\n    'flux_5': ['min', 'max', 'mean', 'median', 'std','skew'],\n    \n    'abs_flux_1': ['median', 'skew'],\n    'abs_flux_2': ['median', 'skew'],\n    'abs_flux_3': ['skew'],\n    'abs_flux_4': ['skew'],\n    'abs_flux_5': ['mean', 'median'],\n    'magnitude_0': ['sum'],\n    'magnitude_1': ['min', 'max', 'mean'],\n    'magnitude_2': ['min', 'max', 'mean'],\n    'magnitude_3': ['min', 'max'],\n    \n    'flux_ratio_sq_0':['sum'],\n    'flux_ratio_sq_4':['skew'],\n    'flux_ratio_sq_5':['sum'],\n    'flux_by_flux_ratio_sq_0':['sum'],\n    'flux_by_flux_ratio_sq_1':['skew'],\n    'flux_by_flux_ratio_sq_5':['skew'],\n    'flux_1_err': ['min', 'max'],\n\n    \"flux_henkaritsu\" : ['min', 'median'],\n    \n    \"flux_sabun_diff\": ['skew'],\n    \"mag_sabun_diff\": ['std'],\n    \"mag_sabun\": ['min', 'std', \"sum\"],\n    \"fl_ratio_sabun\": ['min', 'skew'],\n\n}\n\nagg_train = train_set.groupby('object_id').agg(aggs)\nnew_columns = [\n    k + '_' + agg for k in aggs.keys() for agg in aggs[k]\n]\n\nagg_train.columns = new_columns\n\nagg_train['magnitude_diff_by_mjd'] = (agg_train['magnitude_max'] - agg_train['magnitude_min'])/ agg_train['mjd_diff_mean']\nagg_train['magnitude_dif2'] = (agg_train['magnitude_max'] - agg_train['magnitude_min']) / agg_train['magnitude_mean']\nagg_train['flux_w_mean'] = agg_train['flux_by_flux_ratio_sq_sum'] / agg_train['flux_ratio_sq_sum']\nagg_train['flux_dif3'] = (agg_train['flux_max'] - agg_train['flux_min']) / agg_train['flux_w_mean']\nagg_train[\"flux_dif3_expo2\"] = np.power(agg_train['flux_dif3'], 2) \n\nagg_train[\"flux_0_median_expo2\"] = np.power(agg_train['flux_0_median'], 2)\nagg_train[\"flux_1_median_expo2\"] = np.power(agg_train['flux_1_median'], 2)\nagg_train[\"flux_5_median_expo2\"] = np.power(agg_train['flux_5_median'], 2)\n\nagg_train['magnitude_1_diff_by_mjd'] = (agg_train['magnitude_1_max'] - agg_train['magnitude_1_min'])/ agg_train['mjd_diff_mean']\nagg_train['magnitude_2_diff_by_mjd'] = (agg_train['magnitude_2_max'] - agg_train['magnitude_2_min'])/ agg_train['mjd_diff_mean']\nagg_train['magnitude_3_diff_by_mjd'] = (agg_train['magnitude_3_max'] - agg_train['magnitude_3_min'])/ agg_train['mjd_diff_mean']\ndel agg_train['magnitude_1_max'], agg_train['magnitude_1_min']\ndel agg_train['magnitude_2_max'], agg_train['magnitude_2_min']\ndel agg_train['magnitude_3_max'], agg_train['magnitude_3_min']\n\nagg_train =get_flux_decays(agg_train)\n\ndel train_set\n#del agg_train['flux_max']\ndel agg_train['flux_min'], agg_train['flux_mean'], agg_train['flux_std'], agg_train['flux_median']\ndel agg_train[\"magnitude_min\"], agg_train[\"magnitude_max\"]\n\nprint(gc.collect())\nagg_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"170925efad31dbe7a10f705bc9acbc3ee03e37c7"},"cell_type":"code","source":"full_train = agg_train.reset_index().merge(\n    right=train_set_meta, how='outer', on='object_id')\n\nif 'target' in full_train:\n    y = full_train['target']\n    del full_train['target']\nclasses = sorted(y.unique())\n\nclass_weight = {\n    c: 1 for c in classes\n}\nfor c in [64, 15]:\n    class_weight[c] = 2\n\nprint('Unique classes : ', classes)\n\ndel full_train[\"ra\"], full_train[\"decl\"], full_train[\"gal_l\"], full_train[\"gal_b\"], full_train[\"ddf\"]\nprint(full_train.shape)\n###\nfrom keras.utils import to_categorical\n\nunique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n        \ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\ny_categorical = to_categorical(y_map)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"440f437462a7fb7184ccae357401cd1f369a769f"},"cell_type":"markdown","source":"ft = full_train.copy()"},{"metadata":{"trusted":true,"_uuid":"e96b52a903edc8dc583d17a45442f29e4e161d0d"},"cell_type":"code","source":"#full_train = ft.copy()\n\ndef get_luminosity_features(df):\n    df[\"sqrd_rederr\"] = np.power(df['hostgal_photoz_err'], 2.0)\n    df[\"red_max_err\"] = df[\"flux_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_0_max\"] = df[\"flux_0_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_1_max\"] = df[\"flux_1_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_2_max\"] = df[\"flux_2_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_3_max\"] = df[\"flux_3_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_4_max\"] = df[\"flux_4_max\"] * df[\"sqrd_rederr\"]\n    df[\"sqrd_rederr_5_max\"] = df[\"flux_5_max\"] * df[\"sqrd_rederr\"] \n    df[\"rederr_1_0_max_diff\"] = df[\"sqrd_rederr_1_max\"] - df[\"sqrd_rederr_0_max\"]\n    df[\"rederr_2_1_max_diff\"] = df[\"sqrd_rederr_2_max\"] - df[\"sqrd_rederr_1_max\"]\n    df[\"rederr_3_2_max_diff\"] = df[\"sqrd_rederr_3_max\"] - df[\"sqrd_rederr_2_max\"]\n    df[\"rederr_4_3_max_diff\"] = df[\"sqrd_rederr_4_max\"] - df[\"sqrd_rederr_3_max\"]\n    df[\"rederr_5_4_max_diff\"] = df[\"sqrd_rederr_5_max\"] - df[\"sqrd_rederr_4_max\"]\n    df[\"rederr_4_3_max_by\"] = df[\"sqrd_rederr_4_max\"] / df[\"rederr_4_3_max_diff\"]\n    df[\"rederr_3_2_max_by\"] = df[\"sqrd_rederr_3_max\"] / df[\"rederr_3_2_max_diff\"]\n    df[\"rederr_2_1_max_by\"] = df[\"sqrd_rederr_2_max\"] / df[\"rederr_2_1_max_diff\"]\n    df[\"rederr_1_0_max_by\"] = df[\"sqrd_rederr_1_max\"] / df[\"rederr_1_0_max_diff\"]\n    df[\"rederr__0_max_diff\"] = df[\"red_max_err\"] - df[\"sqrd_rederr_0_max\"]\n    df[\"rederr__1_max_diff\"] = df[\"red_max_err\"] - df[\"sqrd_rederr_1_max\"]\n    df[\"rederr__2_max_diff\"] = df[\"red_max_err\"] - df[\"sqrd_rederr_2_max\"]\n    del df[\"sqrd_rederr\"]\n    del df[\"sqrd_rederr_3_max\"], df[\"sqrd_rederr_1_max\"], df[\"sqrd_rederr_2_max\"], \n    del df[\"sqrd_rederr_4_max\"], df[\"sqrd_rederr_5_max\"], df[\"sqrd_rederr_0_max\"]\n    del df[\"rederr_1_0_max_diff\"], df[\"rederr_4_3_max_diff\"], df[\"rederr_3_2_max_diff\"], df[\"rederr_2_1_max_diff\"],\n\n    ###\n    df[\"sqrd_redshift\"] = np.power(df['hostgal_photoz'], 2.0)\n    df[\"sqrd_red_2_median\"] = df[\"flux_2_median\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_4_mean\"] = df[\"flux_4_mean\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_5_mean\"] = df[\"flux_5_mean\"] * df[\"sqrd_redshift\"] \n    df[\"sqrd_red_0_max\"] = df[\"flux_0_max\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_1_max\"] = df[\"flux_1_max\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_2_max\"] = df[\"flux_2_max\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_3_max\"] = df[\"flux_3_max\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_4_max\"] = df[\"flux_4_max\"] * df[\"sqrd_redshift\"]\n    df[\"sqrd_red_5_max\"] = df[\"flux_5_max\"] * df[\"sqrd_redshift\"]   \n    df[\"abs_lumino_1_median\"] = df[\"abs_flux_1_median\"] * 4 * 3.14 ** df['distmod']\n    df[\"abs_lumino_1_skew\"] = df[\"abs_flux_1_skew\"] * 4 * 3.14 ** df['distmod']\n    df[\"abs_lumino_4_skew\"] = df[\"abs_flux_4_skew\"] * 4 * 3.14 ** df['distmod']\n    ######\n    df[\"dtd_red_min\"] = df[\"flux_detected_min\"] * df[\"sqrd_redshift\"]\n    df[\"dtd_red_median\"] = df[\"flux_detected_median\"] * df[\"sqrd_redshift\"]\n    df[\"dtd_red_std\"] = df[\"flux_detected_std\"] * df[\"sqrd_redshift\"]\n    df[\"dtd_red_skew\"] = df[\"flux_detected_skew\"] * df[\"sqrd_redshift\"]\n    df[\"dtd_red_2_median\"] = df[\"sqrd_red_2_median\"] - df[\"dtd_red_median\"]\n    df[\"abs_dtd_red_1_skew\"] = df[\"abs_lumino_1_skew\"] / df[\"dtd_red_skew\"]\n    df[\"abs_dtd_red_4_skew\"] = df[\"abs_lumino_4_skew\"] / df[\"dtd_red_skew\"]\n    ######    \n    df[\"red_max\"] = df[\"flux_max\"] * df[\"sqrd_redshift\"]\n    df[\"red__0_max_diff\"] = df[\"red_max\"] - df[\"sqrd_red_0_max\"]\n    df[\"red__1_max_diff\"] = df[\"red_max\"] - df[\"sqrd_red_1_max\"]\n    df[\"red__2_max_diff\"] = df[\"red_max\"] - df[\"sqrd_red_2_max\"]\n    del df[\"flux_max\"], df[\"red_max\"], df[\"sqrd_redshift\"]\n\n    df[\"red_2_1_max_diff\"] = df[\"sqrd_red_2_max\"] - df[\"sqrd_red_1_max\"]\n    df[\"red_3_2_max_diff\"] = df[\"sqrd_red_3_max\"] - df[\"sqrd_red_2_max\"]\n    df[\"red_5_4_max_diff\"] = df[\"sqrd_red_5_max\"] - df[\"sqrd_red_4_max\"]\n    del df[\"sqrd_red_3_max\"]\n    ###\n    \n    del df[\"flux_0_min\"], df[\"flux_1_min\"], df[\"flux_2_min\"], df[\"flux_3_min\"], df[\"flux_4_min\"], df[\"flux_5_min\"]\n    del df[\"mwebv\"]\n    del df[\"flux_1_median\"], df[\"flux_2_median\"], df[\"flux_3_median\"], df[\"flux_4_median\"], df[\"flux_5_median\"]\n    del df[\"flux_3_mean\"], df[\"flux_4_max\"]\n    del df['flux_ratio_sq_sum'], df['flux_by_flux_ratio_sq_sum']\n    del df[\"flux_0_median\"], df[\"flux_0_mean\"], df[\"flux_0_std\"]\n    del df[\"flux_1_mean\"], df[\"flux_1_max\"],\n    del df[\"flux_2_max\"], df[\"flux_2_mean\"], df[\"flux_2_std\"],\n    del df[\"flux_3_max\"], df[\"flux_3_std\"],\n    del df[\"flux_4_std\"], df[\"flux_4_mean\"], \n    del df[\"flux_5_max\"], df[\"flux_5_mean\"], df[\"flux_5_skew\"] \n    del df[\"flux_w_mean\"],\n    del df[\"magnitude_mean\"],\n    ##########\n    return df\n\nfull_train = get_luminosity_features(full_train)\n#####\ndel full_train['object_id'], full_train['distmod'], full_train['hostgal_specz']\n\ntrain_mean = full_train.mean(axis=0)\nfull_train.fillna(train_mean, inplace=True)\nprint(full_train.shape)\nfull_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99c63209862334bfb3bbcadfaaceb01d65c56341","scrolled":true},"cell_type":"code","source":"def lgb_multi_weighted_logloss(y_true, y_preds):\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n    y_ohe = pd.get_dummies(y_true)\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    y_p_log = np.log(y_p)\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return 'wloss', loss, False\n\ndef multi_weighted_logloss(y_true, y_preds):\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds\n    y_ohe = pd.get_dummies(y_true)\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    y_p_log = np.log(y_p)\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss\n\nlgb_params = {\n            'device': 'cpu', \n            'objective': 'multiclass', \n            'num_class': 14, \n            'boosting_type': 'gbdt', \n            'n_jobs': -1, \n            'max_depth': 7, \n            'n_estimators': 500, \n            'subsample_freq': 2, \n            'subsample_for_bin': 5000, \n            'min_data_per_group': 100, \n            'max_cat_to_onehot': 4, \n            'cat_l2': 1.0, \n            'cat_smooth': 59.5, \n            'max_cat_threshold': 32, \n            'metric_freq': 10, \n            'verbosity': -1, \n            'metric': 'multi_logloss', \n            'xgboost_dart_mode': False, \n            'uniform_drop': False, \n            'colsample_bytree': 0.5, \n            'drop_rate': 0.173, \n            'learning_rate': 0.0267, \n            'max_drop': 5, \n            'min_child_samples': 10, \n            'min_child_weight': 100.0, \n            'min_split_gain': 0.1, \n            'num_leaves': 7, \n            'reg_alpha': 0.1, \n            'reg_lambda': 0.00023, \n            'skip_drop': 0.44, \n            'subsample': 0.75\n}\n\n# Compute weights\nw = y.value_counts()\nweights = {i : np.sum(w) / w[i] for i in w.index}\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1111)\nclfs = []\nimportances = pd.DataFrame()\noof_preds = np.zeros((len(full_train), len(classes)))\n\nfor fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n    trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n    val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n    \n    clf = lgb.LGBMClassifier(**lgb_params)\n    clf.fit(\n        trn_x, trn_y,\n        eval_set=[(trn_x, trn_y), (val_x, val_y)],\n        eval_metric=lgb_multi_weighted_logloss,\n        verbose=100,\n        early_stopping_rounds=50,\n        sample_weight=trn_y.map(weights)\n    )\n    oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n    print(multi_weighted_logloss(val_y, clf.predict_proba(val_x, num_iteration=clf.best_iteration_)))\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = full_train.columns\n    imp_df['gain'] = clf.feature_importances_\n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    clfs.append(clf)\n\nprint('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_true=y, y_preds=oof_preds))\n\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 24))\nsns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ed30aaee51d1c8f6fe737cdcfefb866e13a3890"},"cell_type":"code","source":"# http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dedb4c89effb2edf51dab0ffc31ad91b8e0b4a22","scrolled":true},"cell_type":"code","source":"import itertools\nfrom sklearn.metrics import confusion_matrix\n\n# Compute confusion matrix\ncnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1))\nnp.set_printoptions(precision=2)\n###\nsample_sub = pd.read_csv('../input/sample_submission.csv')\nclass_names = list(sample_sub.columns[1:-1])\ndel sample_sub;gc.collect()\n\n# Plot non-normalized confusion matrix\nplt.figure(figsize=(12,12))\nfoo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True,\n                      title='Confusion matrix')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf2c1bfb818746ed7b00df98e4bbe42af25bfb9e"},"cell_type":"code","source":"# https://www.kaggle.com/c/PLAsTiCC-2018/discussion/72104\n\ndef GenUnknown(data):\n    return ((((((data[\"mymedian\"]) + (((data[\"mymean\"]) / 2.0)))/2.0)) + (((((1.0) - (((data[\"mymax\"]) * (((data[\"mymax\"]) * (data[\"mymax\"]))))))) / 2.0)))/2.0)\n\nfeats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',\n         'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',\n         'class_92', 'class_95']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"566622d51a01e7f482b3446b39a99c4a1244c590"},"cell_type":"code","source":"import time\n\nstart = time.time()\nchunks = 5000000\nfor i_c, df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)):\n\n    df = get_features_from_mjd(df)\n    df = passbandSplit(df)\n    \n    df['flux_ratio_sq'] = np.power(df['flux'] / df['flux_err'], 2.0)\n    df['flux_by_flux_ratio_sq'] = df['flux'] * df['flux_ratio_sq']\n    \n    df[\"magnitude\"] = -2.5 * np.log(df[\"flux\"]).fillna(0)\n    df[\"magnitude_0\"] = -2.5 * np.log(df[\"flux_0\"]).fillna(0)\n    df[\"magnitude_1\"] = -2.5 * np.log(df[\"flux_1\"]).fillna(0)\n    df[\"magnitude_2\"] = -2.5 * np.log(df[\"flux_2\"]).fillna(0)\n    df[\"magnitude_3\"] = -2.5 * np.log(df[\"flux_3\"]).fillna(0)\n    \n    df = sabun_henkaritsu_cumsum(df)\n    \n    # Group by object id\n    agg_test = df.groupby('object_id').agg(aggs)\n    agg_test.columns = new_columns\n    \n    agg_test['magnitude_diff_by_mjd'] = (agg_test['magnitude_max'] - agg_test['magnitude_min'])/ agg_test['mjd_diff_mean']\n    agg_test['magnitude_dif2'] = (agg_test['magnitude_max'] - agg_test['magnitude_min']) / agg_test['magnitude_mean']\n    agg_test['flux_w_mean'] = agg_test['flux_by_flux_ratio_sq_sum'] / agg_test['flux_ratio_sq_sum']\n    agg_test['flux_dif3'] = (agg_test['flux_max'] - agg_test['flux_min']) / agg_test['flux_w_mean']\n    agg_test[\"flux_dif3_expo2\"] = np.power(agg_test['flux_dif3'], 2) \n\n    agg_test[\"flux_0_median_expo2\"] = np.power(agg_test['flux_0_median'], 2)\n    agg_test[\"flux_1_median_expo2\"] = np.power(agg_test['flux_1_median'], 2)\n    agg_test[\"flux_5_median_expo2\"] = np.power(agg_test['flux_5_median'], 2)\n    \n    agg_test['magnitude_1_diff_by_mjd'] = (agg_test['magnitude_1_max'] - agg_test['magnitude_1_min'])/ agg_test['mjd_diff_mean']\n    agg_test['magnitude_2_diff_by_mjd'] = (agg_test['magnitude_2_max'] - agg_test['magnitude_2_min'])/ agg_test['mjd_diff_mean']\n    agg_test['magnitude_3_diff_by_mjd'] = (agg_test['magnitude_3_max'] - agg_test['magnitude_3_min'])/ agg_test['mjd_diff_mean']\n    del agg_test['magnitude_1_max'], agg_test['magnitude_1_min']\n    del agg_test['magnitude_2_max'], agg_test['magnitude_2_min']\n    del agg_test['magnitude_3_max'], agg_test['magnitude_3_min']\n    \n    agg_test =get_flux_decays(agg_test)\n    #\n    \n#    del agg_test['flux_max'],\n    del agg_test['flux_min'], agg_test['flux_mean'], agg_test['flux_std'], agg_test['flux_median']\n    del agg_test[\"magnitude_min\"], agg_test[\"magnitude_max\"]\n    \n    # Merge with meta data\n    full_test = agg_test.reset_index().merge(\n        right=test_set_meta, how='left', on='object_id')\n    \n    del full_test[\"ra\"], full_test[\"decl\"], full_test[\"gal_l\"], full_test[\"gal_b\"], full_test[\"ddf\"]\n\n    full_test = get_luminosity_features(full_test)    \n    full_test = full_test.fillna(train_mean)\n    \n    # Make predictions\n    preds = None\n    for clf in clfs:\n        if preds is None:\n            preds = clf.predict_proba(full_test[full_train.columns]) / folds.n_splits\n        else:\n            preds += clf.predict_proba(full_test[full_train.columns]) / folds.n_splits\n    \n   # Compute preds_99 as the proba of class not being any of the others\n    # preds_99 = 0.1 gives 1.769\n    preds_99 = np.ones(preds.shape[0])\n    for i in range(preds.shape[1]):\n        preds_99 *= (1 - preds[:, i])\n    \n    # Store predictions\n    preds_df = pd.DataFrame(preds, columns=['class_' + str(s) for s in clfs[0].classes_])\n    preds_df['object_id'] = full_test['object_id']\n    preds_df['class_99'] = 0.14 * preds_99 / np.mean(preds_99) \n    \n    ##########\n    y = pd.DataFrame()\n    y['mymean'] = preds_df[feats].mean(axis=1)\n    y['mymedian'] = preds_df[feats].median(axis=1)\n    y['mymax'] = preds_df[feats].max(axis=1)\n\n    preds_df['class_99'] = GenUnknown(y)\n    ##########\n    \n    if i_c == 0:\n        preds_df.to_csv('predictions.csv',  header=True, mode='a', index=False)\n    else: \n        preds_df.to_csv('predictions.csv',  header=False, mode='a', index=False)\n        \n    del agg_test, full_test, preds_df, preds\n    gc.collect()\n    \n    if (i_c + 1) % 10 == 0:\n        print('%15d done in %5.1f' % (chunks * (i_c + 1), (time.time() - start) / 60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f36a6cedea219c2a25204a0f43e73e099cc8e16"},"cell_type":"code","source":"z = pd.read_csv('predictions.csv')\n\nprint(z.groupby('object_id').size().max())\nprint((z.groupby('object_id').size() > 1).sum())\n\nz = z.groupby('object_id').mean()\n\nz.to_csv('great_trust_cv_predictions.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}