{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"288beff0e5823d4b1f8ff1761cb2ed57c6b1756b"},"cell_type":"code","source":"DATA_DIR = '../input/'\ntrain_meta = pd.read_csv(DATA_DIR + '/training_set_metadata.csv')\ntest_meta = pd.read_csv(DATA_DIR + '/test_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53b546596ee7e64f62f1b5fc3d0157d58594d8cc"},"cell_type":"code","source":"display(train_meta.shape)\ndisplay(train_meta.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"add9ad0e2b70b3f188f36559c2ab82f50a0bc8b3"},"cell_type":"code","source":"display(test_meta.shape)\ndisplay(test_meta.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7efb9bedec04f2bbce6c5545a99c2bbba2bb2e34"},"cell_type":"markdown","source":"**Naive benchmark from kernel https://www.kaggle.com/kyleboone/naive-benchmark-galactic-vs-extragalactic**"},{"metadata":{"trusted":true,"_uuid":"63177df5282f9d694c527af6425df695fd2c825d"},"cell_type":"code","source":"labels = np.hstack([np.unique(train_meta['target']), [99]])\ntarget_map = {j:i for i, j in enumerate(labels)}\ntarget_ids = [target_map[i] for i in train_meta['target']]\ntrain_meta['target_id'] = target_ids\n\n# Build the flat probability arrays for both the galactic and extragalactic groups\ngalactic_cut = train_meta['hostgal_photoz'] == 0\ngalactic_data = train_meta[galactic_cut]\nextragalactic_data = train_meta[~galactic_cut]\n\ngalactic_classes = np.unique(galactic_data['target_id'])\nextragalactic_classes = np.unique(extragalactic_data['target_id'])\n\nprint('galactic_classes = ', galactic_classes)\nprint('extragalactic_classes = ', extragalactic_classes)\n\n# Add class 99 (target_id = 14) to both groups.\ngalactic_classes = np.append(galactic_classes, 14)\nextragalactic_classes = np.append(extragalactic_classes, 14)\n\ngalactic_probabilities = np.zeros(15)\ngalactic_probabilities[galactic_classes] = 1. / len(galactic_classes)\nextragalactic_probabilities = np.zeros(15)\nextragalactic_probabilities[extragalactic_classes] = 1. / len(extragalactic_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32a28a95591fcd365ed17dec08ebafe6dff11b89"},"cell_type":"code","source":"# Apply this naive prediction to a table\ndef do_prediction(table):\n    probs = []\n    for index, row in tqdm.tqdm(table.iterrows(), total=len(table)):\n        if row['hostgal_photoz'] == 0:\n            prob = galactic_probabilities\n        else:\n            prob = extragalactic_probabilities\n        probs.append(prob)\n    return np.array(probs)\n\npred = do_prediction(train_meta)\ntest_pred = do_prediction(test_meta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a30e83642ef43d89da7dde8cbc9601e9cd14c3d0"},"cell_type":"code","source":"train_pred = pd.DataFrame(index=train_meta['object_id'], data=pred, columns=['class_%d' % i for i in labels])\ntest_pred = pd.DataFrame(index=test_meta['object_id'], data=test_pred, columns=['class_%d' % i for i in labels])\nnaive_path = 'naive.csv'\ntrain_pred.to_csv('train_' + naive_path)\ntest_pred.to_csv(naive_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc486161d21d3d440108e2a20d8851c87bf848a5"},"cell_type":"markdown","source":"**Lightgbm on meta from kernel https://www.kaggle.com/johnfarrell/plasticc-2018-metadata-simple-eda**"},{"metadata":{"trusted":true,"_uuid":"91327bc49f23433c54f38c5bd2faa86d9dd95d10"},"cell_type":"code","source":"target = train_meta['target'].values.copy()\ndel train_meta['target']\ntrain_ids = train_meta['object_id'].copy()\ntest_ids = test_meta['object_id'].copy()\ndel train_meta['object_id'], test_meta['object_id']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c90713139ad4cb08cd213ef589374f368b4fa20d"},"cell_type":"markdown","source":"**Class Weights**\n\n* by https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194#397146"},{"metadata":{"trusted":true,"_uuid":"403a56f6ca285cc73b84f0fcfc8e998d377e4984"},"cell_type":"code","source":"labels2weights = {x:1 for x in np.unique(target)}\nlabels2weights[15] = 2\nlabels2weights[64] = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"667180f50035260a4998bf112fe38ee7aadb6112"},"cell_type":"code","source":"train_mask = train_meta['hostgal_photoz'] == 0\ntest_mask = test_meta['hostgal_photoz'] == 0\nsum(train_mask), sum(test_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"809a85a8813a0408b9f97a48d016648af7da3578"},"cell_type":"code","source":"round_params = dict(num_boost_round = 20000,\n                    early_stopping_rounds = 200,\n                    verbose_eval = 50)\nparams = {\n    \"objective\": \"multiclass\",\n    \"metric\": \"multi_logloss\",\n    #\"num_class\": len(np.unique(y)),\n    #\"two_round\": True,\n    \"num_leaves\" : 30,\n    \"min_child_samples\" : 30,\n    \"learning_rate\" : 0.03,\n    \"feature_fraction\" : 0.75,\n    \"bagging_fraction\" : 0.75,\n    \"bagging_freq\" : 1,\n    \"seed\" : 42,\n    \"lambda_l2\": 1e-2,\n    \"verbosity\" : -1\n}\n\ndef lgb_cv_train(X, target, X_test, \n                 params=params, round_params=round_params):\n    print('X', X.shape, 'target', target.shape, 'X_test', X_test.shape)\n    labels = np.unique(target)\n    print('unique labels', labels)\n    \n    ys2labels = dict(enumerate(labels))\n    labels2ys = dict(map(reversed, enumerate(labels)))\n    ys = np.array(list(map(labels2ys.get, target)))\n    weights = np.array(list(map(labels2weights.get, target)))\n    \n    params['num_class'] = len(labels)\n    cv_raw = lgb.cv(\n        params, \n        lgb.Dataset(X, label=ys, weight=weights),\n        nfold=5, \n        **round_params\n    )\n    best_round = np.argmin(cv_raw['multi_logloss-mean'])\n    best_score = cv_raw['multi_logloss-mean'][best_round]\n    print(f'best_round: {best_round}', f'best_score: {best_score}')\n    model = lgb.train(\n        params, \n        lgb.Dataset(X, label=ys, weight=weights), \n        num_boost_round=best_round, \n    )\n    pred = model.predict(X_test)\n    pred_labels = pd.DataFrame(\n        {f'class_{c}': pred[:, i] for i,c in enumerate(labels)}\n    )\n    pred_train = model.predict(X)\n    pred_train_labels = pd.DataFrame(\n        {f'class_{c}': pred_train[:, i] for i,c in enumerate(labels)}\n    )\n    res = dict(\n        model=model,\n        best_round=best_round,\n        best_score=best_score,\n        pred_labels=pred_labels,\n        pred_train_labels=pred_train_labels\n    )\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bae8e921f10fb12626d4914a2986773aeabd252"},"cell_type":"code","source":"feat_extra_li = ['hostgal_specz', 'hostgal_photoz', 'hostgal_photoz_err', 'distmod']\nfeat_gal_cols = ['ra', 'decl', 'gal_l', 'gal_b', 'ddf', 'mwebv']\nfeat_extra_cols = feat_gal_cols + feat_extra_li","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"008b1e54d657109ebf34bd5d4e5f781709b37b0b"},"cell_type":"code","source":"np.unique(target[train_mask]), np.unique(target[~train_mask])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d2d7634b59fe25181263ce90776948e20fa2639"},"cell_type":"code","source":"%%time\nres_gal = lgb_cv_train(\n    train_meta.loc[train_mask, feat_gal_cols], \n    target[train_mask], \n    test_meta.loc[test_mask, feat_gal_cols]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072041e8882149861cd80523b04350f2670b5feb"},"cell_type":"code","source":"n_gal = res_gal['pred_labels'].shape[1]\nres_gal['pred_labels'] = res_gal['pred_labels'] * n_gal/(n_gal+1)\nres_gal['pred_labels']['class_99'] = 1/(n_gal+1)\n# res_gal['pred_labels'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4937b5d6be5e46a99648371299aa0ff1c168e9ac"},"cell_type":"code","source":"%%time\nres_extra = lgb_cv_train(\n    train_meta.loc[~train_mask, feat_extra_cols], \n    target[~train_mask], \n    test_meta.loc[~test_mask, feat_extra_cols]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36a33bdba0432cb83c255cf036c1cc73b13566c1"},"cell_type":"code","source":"n_extra = res_extra['pred_labels'].shape[1]\nres_extra['pred_labels'] = res_extra['pred_labels'] * n_extra/(n_extra+1)\nres_extra['pred_labels']['class_99'] = 1/(n_extra+1)\n# res_extra['pred_labels'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0278c56ac63d6f64e009419eb029765bc310d02e"},"cell_type":"code","source":"train_pred = pd.DataFrame(index=train_ids, columns=['class_%d' % i for i in labels])\ntrain_pred[:] = 0\nfor c in res_gal['pred_train_labels'].columns:\n    train_pred.loc[train_mask.values, c] = res_gal['pred_train_labels'][c].values\nfor c in res_extra['pred_train_labels'].columns:\n    train_pred.loc[~train_mask.values, c] = res_extra['pred_train_labels'][c].values\nlgb_meta_path = 'lgb_meta.csv'\ntrain_pred.to_csv('train_' + lgb_meta_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ada4d3f34312a721e8722b22095204b8997dada"},"cell_type":"code","source":"test_pred = pd.DataFrame(index=test_ids, columns=['class_%d' % i for i in labels])\ntest_pred[:] = 0\nfor c in res_gal['pred_labels'].columns:\n    test_pred.loc[test_mask.values, c] = res_gal['pred_labels'][c].values\nfor c in res_extra['pred_labels'].columns:\n    test_pred.loc[~test_mask.values, c] = res_extra['pred_labels'][c].values\ntest_pred.to_csv(lgb_meta_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"032102b72cd54c98e27f8d577b5a981161ea5eed"},"cell_type":"markdown","source":"**Scores calculation**"},{"metadata":{"_uuid":"2de5bbf9b96b46181ead37d354660bbb65b13771"},"cell_type":"markdown","source":"*log_loss from sklearn.metrics*"},{"metadata":{"trusted":true,"_uuid":"a108a618ae571c68c2f8834ea646b250778fa6e5"},"cell_type":"code","source":"from sklearn.metrics import log_loss\n\nweights = np.array(list(map(labels2weights.get, target)))\nlabels = list(labels2weights.keys()) + [99]\n\nnaive = pd.read_csv('train_' + naive_path)\ndel naive['object_id']\nscore_naive = log_loss(target, naive, eps=1e-15, normalize=True, sample_weight=weights, labels=labels)\nlgb_meta = pd.read_csv('train_' + lgb_meta_path)\ndel lgb_meta['object_id']\nscore_lgb_meta = log_loss(target, lgb_meta, eps=1e-15, normalize=True, sample_weight=weights, labels=labels)\n\nprint('score_naive_sklearn = ', score_naive)\nprint('score_lgb_meta_sklearn = ', score_lgb_meta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"389343b416a8196e95996ae83bfc0b36362edc5c"},"cell_type":"code","source":"def to_df(target):\n    target_df = pd.DataFrame(index=train_ids, columns=['class_%d' % i for i in labels])\n    target_df[:] = 0\n    for i, y in enumerate(target):\n        target_df.iloc[i]['class_%d' % y] = 1\n    return target_df\n\ndef score(target, y_pred, weights, eps=1e-15):\n    targets_df = to_df(target)\n    y_pred = np.clip(y_pred, eps, 1 - eps)\n    sum_by_class = -(targets_df * np.log(y_pred)).sum(axis=0)\n    num_in_class =  np.clip(targets_df.sum(axis=0), 1, float('inf'))\n    score = (weights * sum_by_class / num_in_class).sum(axis=1) / weights.sum(axis=1)\n    return float(score)\n\nclass_weights = {'class_%d' % k: labels2weights[k] for k in labels2weights}\nweights = pd.DataFrame(class_weights, index=[0])\n\nnaive = pd.read_csv('train_' + naive_path).set_index('object_id')\nprint('score_naive = ', score(target, naive, weights))\n\nlgb_meta = pd.read_csv('train_' + lgb_meta_path).set_index('object_id')\nprint('score_lgb_meta = ', score(target, lgb_meta, weights))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"688362269dc03310604599688c2a1be4b259e382"},"cell_type":"markdown","source":"**Submitting**"},{"metadata":{"trusted":true,"_uuid":"a3e7be7242596b4d726cbedb4e530533221ae285"},"cell_type":"code","source":"# !pip install kaggle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6e39f850ef5075550e8080cb027b3c6af20e1c5"},"cell_type":"code","source":"# path = '/tmp/.kaggle/'\n# os.environ['KAGGLE_CONFIG_DIR'] = path\n# basedir = os.path.dirname(path)\n# if not os.path.exists(basedir):\n#     os.makedirs(basedir)\n# filename = path + 'kaggle.json'    \n# if not os.path.isfile(filename):\n#     with open(filename, 'a') as f:\n#         f.write('{\"username\":\"your name\",\"key\":\"your kaggle key\"}')\n# !chmod 600 {filename}\n# print('$HOME=', os.environ['HOME'])\n# print('$KAGGLE_CONFIG_DIR', os.environ['KAGGLE_CONFIG_DIR'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"258575765e66d4557086521035ecea46497479cb"},"cell_type":"code","source":"# !kaggle competitions submit -f {naive_benchmark_path} -m naive_benchmark PLAsTiCC-2018\n# !kaggle competitions submit -f {lgb_meta_path} -m lgb_meta PLAsTiCC-2018","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90e04ace91693175c0eba664aa57e55b9a1d2537"},"cell_type":"code","source":"# submissions = !kaggle competitions submissions -csv PLAsTiCC-2018\n# header = submissions[0].split()\n# naive_submit = [s for s in submissions[2:] if s.startswith(naive_benchmark_path)]\n# lgb_meta_submit = [s for s in submissions[2:] if s.startswith(lgb_meta_path)]\n# score_naive_public = naive_submit[0].split()[-2]\n# score_lgb_meta_public = lgb_meta_submit[0].split()[-2]\n# print('score_naive_public = ', score_naive_public)\n# print('score_lgb_meta_public = ', score_lgb_meta_public)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}