{"cells":[{"metadata":{"_uuid":"fdd3d6875905ec827f8948a0041d793af378deec"},"cell_type":"markdown","source":"### I write a custom function to use the actual evaluation metric for this competition (Matthews Correlation Coefficient)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_columns', 500)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport pyarrow.parquet as pq\nimport os\nprint(os.listdir(\"../input\"))\n\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\n\nfrom sklearn.metrics import mean_squared_error, matthews_corrcoef\n\nfrom sklearn.model_selection import KFold\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport lightgbm as lgb\n\n# Any results you write to the current directory are saved as output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4293166471fbd94cd05a6059d7aaf57dd3de9fc7"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n#     return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"meta_train = pd.read_csv('../input/metadata_train.csv')\n# meta_test = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"598f763b4578ff82f42c6c4e2e521914a2152cfd"},"cell_type":"code","source":"reduce_mem_usage(meta_train)\n# reduce_mem_usage(meta_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ef0550bf2d395193ea1194bd9c5b6791190ebe0"},"cell_type":"markdown","source":"### Very inefficient approach to  start off with -- will clean this up later"},{"metadata":{"trusted":true,"_uuid":"d5b6afb3e16dca5eba020af507b794eba8dbef94"},"cell_type":"code","source":"meta_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a8a51c3c32939728cb78462293542e90e1804c8"},"cell_type":"markdown","source":"### Will someone please give me a custom file parser for Christmas"},{"metadata":{"trusted":true,"_uuid":"1048ae276f934f0b4fc6967bc45108150c488d91"},"cell_type":"code","source":"%%time \n# Start small, load in 500 signals and calculate some basic aggregates (mean, sum)\n# Read in 500 signals\n# Each column contains one signal\nsubset_train_0_500 = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(500)]).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3864d65b7825f79ef18234e826a1a726589c80f"},"cell_type":"markdown","source":"### In line with the discussion (https://www.kaggle.com/c/vsb-power-line-fault-detection/discussion/75373) I have used numpy"},{"metadata":{"trusted":true,"_uuid":"479ab80b02298faaefaa620ee9d1516cdf629d16"},"cell_type":"code","source":"# Function to add a few basic aggregations for the different signal ids\ndef add_cols(df, df_sig):\n    for signal in df['signal_id'].tolist():\n        df.loc[df['signal_id']==signal,'signal_mean'] = np.mean(df_sig[str(signal)])\n        df.loc[df['signal_id']==signal,'signal_sum'] = np.sum(df_sig[str(signal)])\n        df.loc[df['signal_id']==signal,'signal_median'] = np.median(df_sig[str(signal)])\n        df.loc[df['signal_id']==signal,'signal_ptp'] = np.ptp(df_sig[str(signal)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c705802b123bc2b6290f8e81c2e9caf87a26639d"},"cell_type":"code","source":"meta_train['signal_mean'] = 0\nmeta_train['signal_sum'] = 0\nmeta_train['signal_median'] = 0\nmeta_train['signal_ptp'] = 0\nmeta_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b5595ed59fca69f243515cc2759316da39d18df"},"cell_type":"code","source":"meta_train_0_500 = meta_train[:500]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64e324c8caf3e900852f918b817d9407f9d26903"},"cell_type":"code","source":"%%time\nadd_cols(meta_train_0_500,subset_train_0_500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82329d35e01bc7d97d6fa3698e342e68b10e9294"},"cell_type":"code","source":"reduce_mem_usage(meta_train_0_500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab6d10be9ab7e1325d1656d035edb187daa83ea0"},"cell_type":"code","source":"meta_train_0_500.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75d58c315d64eef55e6c2db70bce9d5cce248d70"},"cell_type":"code","source":"meta_train_0_500.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4c67d294b2eef953f410cf729d53d342c05d4f1"},"cell_type":"markdown","source":"### Clearly this isn't a viable solution for the test set, so let's at least get a reasonable CV score from the few training examples we have signal aggregations for, and leave the rest till someone more experienced comes up with a method for handling the rather large datasets"},{"metadata":{"trusted":true,"_uuid":"bf54e4c7c7375e8a881f3b1a61e1647868658257"},"cell_type":"code","source":"target = meta_train_0_500['target']\nmeta_train_0_500.drop('target', axis=1, inplace=True)\nfeatures = meta_train_0_500.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"906528e39f45a1bf893d38e894ce11626d6ebc0a"},"cell_type":"markdown","source":"### Create a custom feval for the Matthews Correlation Coefficient"},{"metadata":{"trusted":true,"_uuid":"f00d5b279e0109217e30e3d1c40a005205cd42f3"},"cell_type":"code","source":"# def lgb_mcc_score(y_hat, data):\n#     y_true = data.get_label()\n#     y_hat = np.round(y_hat)\n#     return 'mcc', matthews_corrcoef(y_true, y_hat), True\n\n# # def lgb_mcc(preds, dtrain):\n# #     THRESHOLD = 0.5\n# #     labels = dtrain.get_label()\n# #     return 'mcc', matthews_corrcoef(labels, preds >= THRESHOLD)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"482341478b3929fe0251bcbb491bf29ba5a55a1a"},"cell_type":"markdown","source":"### Have to fix this"},{"metadata":{"trusted":true,"_uuid":"acdcfdf73a8290b265519eaaa3d3dc462fc63811"},"cell_type":"code","source":"param = {'num_leaves': 60,\n         'min_data_in_leaf': 60, \n         'objective':'binary',\n         'max_depth': -1,\n         'learning_rate': 0.1,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.8,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.8 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'auc',\n         \"lambda_l1\": 0.1,\n         \"random_state\": 42,\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2575600866b399d4277dc8968f0c3eba5ecb7359"},"cell_type":"code","source":"max_iter=5\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ecfaf8dfe962a7405f34ab671fbcdc415d6448e"},"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=42)\noof = np.zeros(len(meta_train_0_500))\n# categorical_columns = [c for c in categorical_columns if c not in ['MachineIdentifier']]\n# features = [c for c in train.columns if c not in ['MachineIdentifier']]\n# predictions = np.zeros(len(test))\n\nfeature_importance_df = pd.DataFrame()\n\nscore = [0 for _ in range(folds.n_splits)]\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(meta_train_0_500.values, target.values)):\n    print(\"Fold No.{}\".format(fold_+1))\n    trn_data = lgb.Dataset(meta_train_0_500.iloc[trn_idx][features],\n                           label=target.iloc[trn_idx],\n#                            categorical_feature = categorical_columns\n                          )\n    val_data = lgb.Dataset(meta_train_0_500.iloc[val_idx][features],\n                           label=target.iloc[val_idx],\n#                            categorical_feature = categorical_columns\n                          )\n    evals_result = {}\n    \n    num_round = 10000\n    clf = lgb.train(param,\n                    trn_data,\n                    num_round,\n                    valid_sets = [trn_data, val_data],\n                    verbose_eval=100,\n                    early_stopping_rounds = 200)\n#                     feval = lgb_mcc_score,\n#                     evals_result = evals_result)\n    \n    oof[val_idx] = clf.predict(meta_train_0_500.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance(importance_type='gain')\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n#     lgb.plot_metric(evals_result, metric='mcc')\n\n    score[fold_] = metrics.roc_auc_score(target.iloc[val_idx], oof[val_idx])\n    if fold_ == max_iter - 1: break\n        \nif (folds.n_splits == max_iter):\n    print(\"CV score: {:<8.5f}\".format(metrics.roc_auc_score(target, oof)))\nelse:\n     print(\"CV score: {:<8.5f}\".format(sum(score) / max_iter))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd460e6ba7fecf544cdd572f2eb425119eaac37"},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(10,10))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5442630b49608584965b082f80108cd58de3e0a4"},"cell_type":"markdown","source":"### Is phase really that insignificant in target prediction? "},{"metadata":{"trusted":true,"_uuid":"fc08ea6185695d4f83fc46f5da2f19e7f9ffbca1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}