{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\"\"\"\nPLAsTiCC Astronomical Classification Feature Extraction\n----------------------------------\n@website https://www.kaggle.com/mithrillion/know-your-objective/\n\nGoal :\n------\nWe are starting with basic modeling technique explained in Oliver's kernal to understand implementation of lightgbms and generating features for big data.\n\nTrain 5 lightgbms on the meta_data + aggregated data\n\nThen go through test data in chunks and generate predictions\n\nNew in this version :\n---------------------\n1. This versions adds some of the Flux calculations made available by MichaelApers https://www.kaggle.com/michaelapers\n    here https://www.kaggle.com/michaelapers/the-plasticc-astronomy-starter-kit\n2. class 99 mean adjustment\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1cbd5fac383acc399ad2832d051b26de00c244d"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport logging\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a17e2226784662f81b6d408d226f95c47b8f071f"},"cell_type":"code","source":"def create_logger():\n    logger_ = logging.getLogger('main')\n    logger_.setLevel(logging.DEBUG)\n    fh = logging.FileHandler('simple_lightgbm.log')\n    fh.setLevel(logging.DEBUG)\n    ch = logging.StreamHandler()\n    ch.setLevel(logging.DEBUG)\n    formatter = logging.Formatter('[%(levelname)s]%(asctime)s:%(name)s:%(message)s')\n    fh.setFormatter(formatter)\n    ch.setFormatter(formatter)\n    # add the handlers to the logger\n    logger_.addHandler(fh)\n    logger_.addHandler(ch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2079d9def0d3d82691d563b78823099b791a05b7"},"cell_type":"code","source":"def get_logger():\n    return logging.getLogger('main')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def lgb_multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    # class_weights taken from Giba's topic : https://www.kaggle.com/titericz\n    # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n    # with Kyle Boone's post https://www.kaggle.com/kyleboone\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return 'wloss', loss, False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17311402d321fac42447af4a899641c7b52d2f81"},"cell_type":"code","source":"def multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    # class_weights taken from Giba's topic : https://www.kaggle.com/titericz\n    # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n    # with Kyle Boone's post https://www.kaggle.com/kyleboone\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    if len(np.unique(y_true)) > 14:\n        classes.append(99)\n        class_weight[99] = 2\n    y_p = y_preds\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a6bcac6a66ca93319b4c620695edaa1915048e0"},"cell_type":"code","source":"def get_aggregations():\n    return {\n        # Dropped mjd aggregations on CPMP advice\n        # see https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69696\n        # 'mjd': ['min', 'max', 'size'],\n        'passband': ['mean', 'std', 'var'],  # ''min', 'max', 'mean', 'median', 'std'],\n        'flux': ['min', 'max', 'mean', 'median', 'std'],\n        'flux_err': ['min', 'max', 'mean', 'median', 'std'],\n        'detected': ['mean'],  # ''min', 'max', 'mean', 'median', 'std'],\n    }\n\n\ndef get_new_columns(aggs):\n    return [k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n\n\ndef add_features_to_agg(df):\n    # CPMP using the following feature was really silliy :)\n    # df['mjd_diff'] = df['mjd_max'] - df['mjd_min']\n    # see https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69696\n    \n    # The others may be useful\n    df['flux_diff'] = df['flux_max'] - df['flux_min']\n    df['flux_dif2'] = (df['flux_max'] - df['flux_min']) / df['flux_mean']\n    df['flux_w_mean'] = df['flux_by_flux_ratio_sq_sum'] / df['flux_ratio_sq_sum']\n    df['flux_dif3'] = (df['flux_max'] - df['flux_min']) / df['flux_w_mean']\n\n    # del df['mjd_max'], df['mjd_min']\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"702569fc04c535b9b7bc10fc4f6f103b53fa5ff0"},"cell_type":"code","source":"def Generate_Features_Chunk(df_, meta_, features, train_mean):\n\n    df_['flux_ratio_sq'] = np.power(df_['flux'] / df_['flux_err'], 2.0)\n    df_['flux_by_flux_ratio_sq'] = df_['flux'] * df_['flux_ratio_sq']\n\n    # Group by object id\n    aggs = get_aggregations()\n\n    aggs = get_aggregations()\n    aggs['flux_ratio_sq'] = ['sum']\n    aggs['flux_by_flux_ratio_sq'] = ['sum']\n\n    new_columns = get_new_columns(aggs)\n\n    agg_ = df_.groupby('object_id').agg(aggs)\n    agg_.columns = new_columns\n\n    agg_ = add_features_to_agg(df=agg_)\n\n    # Merge with meta data\n    full_test = agg_.reset_index().merge(\n        right=meta_,\n        how='left',\n        on='object_id'\n    )\n\n    full_test = full_test.fillna(train_mean)\n    return full_test\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"c4c8bf69a88fcf22caaf102c51f66620cd6501ce"},"cell_type":"code","source":"def main():\n    train = pd.read_csv('../input/training_set.csv')\n    train['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)\n    train['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']\n\n    # train = pd.concat([train, pd.get_dummies(train['passband'], prefix='passband')], axis=1, sort=False)\n\n    aggs = get_aggregations()\n    aggs['flux_ratio_sq'] = ['sum']\n    aggs['flux_by_flux_ratio_sq'] = ['sum']\n\n    # passbands = [f for f in train if 'passband_' in f]\n    # get_logger().info('Passband features : {}'.format(passbands))\n    # for pb in passbands:\n    #     aggs[pb] = ['mean']\n\n    agg_train = train.groupby('object_id').agg(aggs)\n    new_columns = get_new_columns(aggs)\n    agg_train.columns = new_columns\n\n    agg_train = add_features_to_agg(df=agg_train)\n    \n    agg_train.head()\n\n    del train\n    gc.collect()\n\n    meta_train = pd.read_csv('../input/training_set_metadata.csv')\n    meta_train.head()\n\n    full_train = agg_train.reset_index().merge(\n        right=meta_train,\n        how='outer',\n        on='object_id'\n    )\n\n    train_mean = full_train.mean(axis=0)\n    full_train.fillna(train_mean, inplace=True)\n    get_logger().info(full_train.columns)\n    \n    #create feature set for training dataset\n    full_train.to_csv('Training_Features.csv', index=True, float_format='%.6f')\n    \n    meta_test = pd.read_csv('../input/test_set_metadata.csv')\n\n    import time\n\n    start = time.time()\n    chunks = 5000000\n    remain_df = None\n    \n    def the_unique(x):\n        return [x[i] for i in range(len(x)) if x[i] != x[i-1]]\n\n    for i_c, df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)):\n        # Check object_ids\n        # I believe np.unique keeps the order of group_ids as they appear in the file\n        # My belief is wrong (I should have read the doc !)\n        # A big thank you to https://www.kaggle.com/filby89\n        # Use .tolist() is almost 3 times faster than the_unique(df['object_id'].values)\n        unique_ids = the_unique(df['object_id'].tolist())\n        new_remain_df = df.loc[df['object_id'] == unique_ids[-1]].copy()\n\n        if remain_df is None:\n            df = df.loc[df['object_id'].isin(unique_ids[:-1])].copy()\n        else:\n            df = pd.concat([remain_df, df.loc[df['object_id'].isin(unique_ids[:-1])]], axis=0)\n\n        # Create remaining samples df\n        remain_df = new_remain_df\n\n        test_features_df = Generate_Features_Chunk(df_=df,\n                                 meta_=meta_test,\n                                 features=full_train.columns,\n                                 train_mean=train_mean)\n                \n        if i_c == 0:\n            test_features_df.to_csv('Test_Features.csv', header=True, index=False, float_format='%.6f')\n        else:\n            test_features_df.to_csv('Test_Features.csv', header=False, mode='a', index=False, float_format='%.6f')\n\n        del test_features_df\n        gc.collect()\n\n        if (i_c + 1) % 10 == 0:\n            get_logger().info('%15d done in %5.1f' % (chunks * (i_c + 1), (time.time() - start) / 60))\n            print('%15d done in %5.1f' % (chunks * (i_c + 1), (time.time() - start) / 60))\n       # Compute last object in remain_df\n\n    test_features_df = Generate_Features_Chunk(df_=remain_df,\n                                 meta_=meta_test,\n                                 features=full_train.columns,\n                                 train_mean=train_mean)\n\n    test_features_df.to_csv('Test_Features.csv', header=False, mode='a', index=False, float_format='%.6f')\n\nif __name__ == '__main__':\n    gc.enable()\n    create_logger()\n    try:\n        main()\n    except Exception:\n        get_logger().exception('Unexpected Exception Occured')\n        raise","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}