{"cells":[{"metadata":{"_uuid":"02a020976fc15653cc0b767edc88d2ab68ac86e4"},"cell_type":"markdown","source":"The following kernel has been inspired by:\n\nhttps://www.kaggle.com/ogrellier/plasticc-in-a-kernel-meta-and-data\n\nMain addition is structuring the code with an sklearn pipeline."},{"metadata":{"trusted":true,"_uuid":"acf0cde8df2730588bb30e44eb9b537876739731"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport time\n\nimport lightgbm as lgb\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63f25c206248a1ca9f696cd2a74f8744bfb96ba2"},"cell_type":"markdown","source":"# 1. Load and transform data"},{"metadata":{"trusted":true,"_uuid":"42ad3d7e357202c2e62ba21edf4966a22f403a0a"},"cell_type":"code","source":"my_aggs = {\n           'passband': ['mean', 'std', 'var'],\n           'flux': ['min', 'max', 'mean', 'median', 'std'],\n           'flux_err': ['min', 'max', 'mean', 'median', 'std'],\n           'detected': ['mean'],\n           'flux_ratio_sq': ['sum'],\n           'flux_by_flux_ratio_sq':['sum']\n          }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c192e874c1cf1fddd63d6e31ccb82504f35ef812"},"cell_type":"code","source":"def transform_ts(data_ts, aggs):\n    #copy\n    df_ts = data_ts.copy()\n    \n    #add\n    df_ts = df_ts.assign(flux_ratio_sq = np.power(df_ts['flux'] / df_ts['flux_err'], 2.0))\n    df_ts = df_ts.assign(flux_by_flux_ratio_sq = df_ts['flux'] * df_ts['flux_ratio_sq'])\n\n    #aggregate\n    df_ts_agg = df_ts.groupby(['object_id']).agg(aggs)\n    df_ts_agg.columns = ['_'.join((col[0],col[1])) for col in df_ts_agg.columns]\n    df_ts_agg = df_ts_agg.reset_index()\n    \n    return df_ts_agg\n    \ndef transform_ts_chunk(from_file, to_file, aggs, chunk_size=5000000):\n    \n    remain_df=None\n    start = time.time()\n\n    for i, df in enumerate(pd.read_csv(from_file, chunksize=chunk_size, iterator=True)):\n        \n        # set aside data of last object_id (may be implete)\n        remain_id = df.iloc[-1]['object_id']\n        new_remain_df = df[df['object_id'] == remain_id].copy()\n        df = df[~(df['object_id'] == remain_id)].copy()\n        \n        # add remain_df of last iteration if exists. \n        if remain_df is not None:\n            df = pd.concat([remain_df, df], axis=0)\n        remain_df = new_remain_df\n        \n        # apply transformations and save\n        df_ts_agg = transform_ts(df, aggs)\n        if i == 0:\n            df_ts_agg.to_csv(to_file, header=True, index=False)\n        else:\n            df_ts_agg.to_csv(to_file, header=False, index=False, mode='a')\n    \n        # print progress\n        print(\"chunk\", i, \"done in\", int(time.time() - start), \"sec\")\n        start = time.time()\n    \n    # add remaining object\n    df_ts_agg = transform_ts(remain_df, aggs)\n    df_ts_agg.to_csv(to_file, header=False, index=False, mode='a')\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61d8cfdab6e7f2b78a13162e4f5461e8de5f6aa8"},"cell_type":"markdown","source":"1.1 transform train data"},{"metadata":{"_uuid":"3f0d8dcb62315554e613d51d9abdc5ce58cc59d7"},"cell_type":"markdown","source":"1.1.1 aggregate timeseries"},{"metadata":{"trusted":true,"_uuid":"a320e927639a5277c14885a1930577496cdac6cc"},"cell_type":"code","source":"train_ts = pd.read_csv(\"../input/training_set.csv\")\ntrain_ts_agg = transform_ts(train_ts, my_aggs)\ntrain_ts_agg.to_csv(\"training_set_agg.csv\", header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7f7f3be735e697bec08aa81a2c53cfe3008f8ac"},"cell_type":"markdown","source":"1.1.2 merge aggregated timeseries to meta and save"},{"metadata":{"trusted":true,"_uuid":"00efd7f66b53e096aaf61422174c3d1f312b37bb"},"cell_type":"code","source":"train_meta = pd.read_csv(\"../input/training_set_metadata.csv\")\ntrain_total  = train_meta.merge(train_ts_agg, on=['object_id'], how='left')\ntrain_total.to_csv(\"training_total.csv\", header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd4c224aacdab372114c3ca3d6777e9e6f2d3b4d"},"cell_type":"markdown","source":"1.2 transform test ts data in chunks\n\n1.2.1 aggregate timeseries"},{"metadata":{"trusted":true,"_uuid":"43c3c459abe288a6cd2a76ef126adee75f41e1d7"},"cell_type":"code","source":"transform_ts_chunk(\"../input/test_set.csv\", \"test_set_agg.csv\", my_aggs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32fa099e98a3954b2d2a7dd267937ec031e33e3e"},"cell_type":"markdown","source":"1.2.2 merge aggregated timeseries to meta and save"},{"metadata":{"trusted":true,"_uuid":"890134dc386c74dcef78997343e15ae4ac654798"},"cell_type":"code","source":"test_meta = pd.read_csv(\"../input/test_set_metadata.csv\")\ntest_ts_agg = pd.read_csv(\"test_set_agg.csv\")\ntest_total = test_meta.merge(test_ts_agg, on=['object_id'], how='left')\ntest_total.to_csv(\"test_total.csv\", header=True, index=False)\ndel test_ts_agg, test_meta, test_total","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d84a85398a615bfea7cdd1a289e137636d96607"},"cell_type":"markdown","source":"# 2. Train model"},{"metadata":{"trusted":true,"_uuid":"0fa2ae321ed6515946a746a44652572ca2831759"},"cell_type":"code","source":"def lgb_multi_weighted_logloss(y_true, y_preds, scorer=False):\n    \"\"\"@author olivier https://www.kaggle.com/ogrellier\"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n\n    y_ohe = pd.get_dummies(y_true)\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    y_p_log = np.log(y_p)\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return 'wloss', loss, False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db6064683c5a7ae5221b6b592e5ae8f85058062a"},"cell_type":"code","source":"class imputer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        self.means = dict()\n\n    def fit(self, X, y=None):\n        print('Fitting the imputer...')\n                \n        numeric_columns = list(X.select_dtypes(include=[np.number]).columns)\n        for col in numeric_columns:\n            try:\n                self.means[col] = int(X.loc[:,col].mean())\n            except:\n                self.means[col] = 0\n        \n        return self\n\n    def transform(self, X):\n        print('Transforming the data with the imputer...')\n        X_new = X.copy()\n        non_numeric_columns = list(X_new.select_dtypes(exclude=[np.number]).columns)\n        for col in non_numeric_columns:\n            na_value = \"unknown\"\n            X_new[col].fillna(na_value, inplace=True)\n        \n        numeric_columns = list(X_new.select_dtypes(include=[np.number]).columns)\n        for col in numeric_columns:\n            na_value = self.means[col]\n            X_new[col].fillna(na_value, inplace=True)\n        return X_new\n\nclass addFeatures(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        pass\n\n    def fit(self, X, y=None):\n        return self \n\n    def transform(self, X):\n        print('Adding features...')\n        X_new = X.copy()\n        \n        X_new['flux_diff'] = X_new['flux_max'] - X_new['flux_min']\n        X_new['flux_rel_diff'] = (X_new['flux_max'] - X_new['flux_min']) / X_new['flux_mean']\n        X_new['flux_w_mean'] = X_new['flux_by_flux_ratio_sq_sum'] / X_new['flux_ratio_sq_sum']\n        X_new['flux_rel_diff2'] = (X_new['flux_max'] - X_new['flux_min']) / X_new['flux_w_mean']\n        return X_new\n\nclass dropFeatures(BaseEstimator, TransformerMixin):\n    def __init__(self, cols):\n        self.cols = cols\n\n    def fit(self, X, y=None):\n        return self \n\n    def transform(self, X):\n        print('Dropping features...')\n        X_new = X.copy()\n        for col in self.cols:\n            X_new = X_new.drop(col, axis=1)\n        return X_new\n    \nclass customModel(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        self.clfs = []\n        self.lgb_params = {\n            'boosting_type': 'gbdt',\n            'objective': 'multiclass',\n            'num_class': 14,\n            'metric': 'multi_logloss',\n            'learning_rate': 0.03,\n            'subsample': .9,\n            'colsample_bytree': .7,\n            'reg_alpha': .01,\n            'reg_lambda': .01,\n            'min_split_gain': 0.01,\n            'min_child_weight': 10,\n            'n_estimators': 1000,\n            'silent': -1,\n            'verbose': -1,\n            'max_depth': 3\n        }\n        \n    def fit(self, X, y):\n        print('Fitten the model...')\n        folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\n        \n        w = y.value_counts()\n        weights = {i: np.sum(w) / w[i] for i in w.index}\n        \n        for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n            trn_x, trn_y = X.iloc[trn_], y.iloc[trn_]\n            val_x, val_y = X.iloc[val_], y.iloc[val_]\n\n            clf = lgb.LGBMClassifier(**self.lgb_params)\n            clf.fit(\n                trn_x, trn_y,\n                eval_set=[(trn_x, trn_y), (val_x, val_y)],\n                eval_metric=lgb_multi_weighted_logloss,\n                verbose=100,\n                early_stopping_rounds=50,\n                sample_weight=trn_y.map(weights)\n            )\n            self.clfs.append(clf)\n        return self\n        \n    def predict(self, X):\n        print('Predicting...')\n        preds_ = None\n        for clf in self.clfs:\n            if preds_ is None:\n                preds_ = clf.predict_proba(X) / len(self.clfs)\n            else:\n                preds_ += clf.predict_proba(X) / len(self.clfs)\n        return preds_\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b00e8602bc46327b161f89f5417375e4ee0fe1b"},"cell_type":"code","source":"df_train = pd.read_csv(\"training_total.csv\")\nX_train = df_train.drop(['target'],1)\ny_train = df_train['target']\n\nestimator_lgb = Pipeline(steps = [\n        ('imputer', imputer()),\n        ('add_features', addFeatures()),\n        ('drop_features', dropFeatures(['object_id', 'hostgal_specz'])),\n        ('customModel', customModel())])\n\nestimator_lgb.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c33cd1383a5f6fa090fa9e6b20194300ae94cef9"},"cell_type":"markdown","source":"# 3. Predict for test"},{"metadata":{"trusted":true,"_uuid":"635e2bfdb92125d5b299e6efe07372a9bb268a61"},"cell_type":"code","source":"X_test = pd.read_csv(\"test_total.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4012650f88ef8696b7cb9944b85e3616d59544eb"},"cell_type":"code","source":"splits = np.array_split(X_test['object_id'].unique(), 50)\n\nnormalize_rows_to_one = True\n \nfor i, split in enumerate(splits):\n    print(\"split:\", i)\n    chunk = X_test[X_test['object_id'].isin(split)]\n    chunk_pred = estimator_lgb.predict(chunk)\n\n    # prob of class 99 is probability of not other clases\n    preds_99 = np.ones(chunk_pred.shape[0])\n    for j in range(chunk_pred.shape[1]):\n        preds_99 *= (1 - chunk_pred[:, j])\n    preds_99 = np.expand_dims(preds_99,1)\n    preds_99 = 0.14 * preds_99 / np.mean(preds_99)\n    chunk_pred = np.append(chunk_pred, preds_99, axis=1)\n\n    # rescale such that all probs in one row add up to 1.\n    if(normalize_rows_to_one):\n        row_sums = np.expand_dims(np.sum(chunk_pred, axis = 1),1)\n        chunk_pred = chunk_pred/row_sums\n\n    # alles lekker aan elkaar plakken\n    if i==0:\n        y_test_pred = chunk_pred\n    else:\n        y_test_pred = np.append(y_test_pred, chunk_pred, axis=0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b27c081dbf89597c374c3268ec9931a99c9e2e33"},"cell_type":"code","source":"#y_test_pred = y_test_pred.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db1ab48648edb0ffa7744a5582720e777858285a"},"cell_type":"code","source":"targets = estimator_lgb.named_steps.customModel.clfs[0].classes_\ntargets = np.append(targets,'99')\ny_test_pred_df = pd.DataFrame(index=X_test['object_id'], data=y_test_pred, columns=['class_'+i for i in targets])\ny_test_pred_df.reset_index(inplace = True)\ny_test_pred_df.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e51ce0b3c6b5c4c6e7d6b16e8171f40520e88143"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"312179e6d80e701bda3457a20a6824502eff25b9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}