{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Evaluation\nThis notebooks shows how to use `evaluate` to calculate the evaluation metrics for the AMEX competition.\n\n## Install `evaluate`","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install evaluate","metadata":{"execution":{"iopub.status.busy":"2022-06-09T16:07:51.064258Z","iopub.execute_input":"2022-06-09T16:07:51.064789Z","iopub.status.idle":"2022-06-09T16:08:03.019468Z","shell.execute_reply.started":"2022-06-09T16:07:51.064749Z","shell.execute_reply":"2022-06-09T16:08:03.018072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data\n\nFirst, let's load the competition data:","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\ninput_path = Path('/kaggle/input/amex-default-prediction/')\n\ntrain_data = pd.read_csv(\n    input_path / 'train_data.csv',\n    index_col='customer_ID',\n    usecols=['customer_ID', 'P_2'])\n\ntrain_labels = pd.read_csv(input_path / 'train_labels.csv', index_col='customer_ID')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make predictions\nWith the training data we can make some dummy predicitons:","metadata":{}},{"cell_type":"code","source":"ave_p2 = (train_data\n          .groupby('customer_ID')\n          .mean()\n          .rename(columns={'P_2': 'prediction'}))\n\n# Scale the mean P_2 by the max value and take the compliment\nave_p2['prediction'] = 1.0 - (ave_p2['prediction'] / ave_p2['prediction'].max())","metadata":{"execution":{"iopub.status.busy":"2022-06-09T16:14:58.462958Z","iopub.execute_input":"2022-06-09T16:14:58.46357Z","iopub.status.idle":"2022-06-09T16:15:00.094309Z","shell.execute_reply.started":"2022-06-09T16:14:58.463503Z","shell.execute_reply":"2022-06-09T16:15:00.093328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation with `evaluate` \n\nWith `evaluate` anybody can add community metrics (see the [guide](https://huggingface.co/docs/evaluate/creating_and_sharing) in the documentation). We added the AMEX evaluation metric as a community metric so anybody can easily use it which comes with an interactive widget and a metric card:\n\nhttps://huggingface.co/spaces/kaggle/amex\n\nUsing a community metric with `evaluate` is just two lines of code:","metadata":{}},{"cell_type":"code","source":"import evaluate\n\namex_metric = evaluate.load(\"kaggle/amex\")\namex_metric.compute(references=train_labels[\"target\"], predictions=ave_p2[\"prediction\"])","metadata":{"execution":{"iopub.status.busy":"2022-06-09T16:15:07.538981Z","iopub.execute_input":"2022-06-09T16:15:07.539624Z","iopub.status.idle":"2022-06-09T16:15:29.261504Z","shell.execute_reply.started":"2022-06-09T16:15:07.539574Z","shell.execute_reply":"2022-06-09T16:15:29.260666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Original implementation\nWe can verify that this is the same result as with the original implementation provided by the organizers:","metadata":{}},{"cell_type":"code","source":"def amex_metric_original(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-09T16:08:37.570239Z","iopub.execute_input":"2022-06-09T16:08:37.570695Z","iopub.status.idle":"2022-06-09T16:08:37.588803Z","shell.execute_reply.started":"2022-06-09T16:08:37.570664Z","shell.execute_reply":"2022-06-09T16:08:37.587176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(amex_metric_original(train_labels, ave_p2)) ","metadata":{"execution":{"iopub.status.busy":"2022-06-09T16:15:03.678686Z","iopub.execute_input":"2022-06-09T16:15:03.679155Z","iopub.status.idle":"2022-06-09T16:15:05.302628Z","shell.execute_reply.started":"2022-06-09T16:15:03.679113Z","shell.execute_reply":"2022-06-09T16:15:05.301595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Indeed, we get the same result :) ","metadata":{}}]}