{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:09:50.067299Z","iopub.execute_input":"2022-08-02T09:09:50.067709Z","iopub.status.idle":"2022-08-02T09:09:50.074902Z","shell.execute_reply.started":"2022-08-02T09:09:50.067670Z","shell.execute_reply":"2022-08-02T09:09:50.073417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:09:50.422339Z","iopub.execute_input":"2022-08-02T09:09:50.423429Z","iopub.status.idle":"2022-08-02T09:09:50.436528Z","shell.execute_reply.started":"2022-08-02T09:09:50.423378Z","shell.execute_reply":"2022-08-02T09:09:50.435278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When you get 100% correct","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:10000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:10:07.972365Z","iopub.execute_input":"2022-08-02T09:10:07.972760Z","iopub.status.idle":"2022-08-02T09:10:08.189770Z","shell.execute_reply.started":"2022-08-02T09:10:07.972729Z","shell.execute_reply":"2022-08-02T09:10:08.188698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When your prediction on default is 50% correct","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:5000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:09:56.810910Z","iopub.execute_input":"2022-08-02T09:09:56.811419Z","iopub.status.idle":"2022-08-02T09:09:57.073814Z","shell.execute_reply.started":"2022-08-02T09:09:56.811373Z","shell.execute_reply":"2022-08-02T09:09:57.072630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:8000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:10:38.806226Z","iopub.execute_input":"2022-08-02T09:10:38.806628Z","iopub.status.idle":"2022-08-02T09:10:39.007475Z","shell.execute_reply.started":"2022-08-02T09:10:38.806591Z","shell.execute_reply":"2022-08-02T09:10:39.006336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:8500, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:10:54.142521Z","iopub.execute_input":"2022-08-02T09:10:54.143704Z","iopub.status.idle":"2022-08-02T09:10:54.359184Z","shell.execute_reply.started":"2022-08-02T09:10:54.143661Z","shell.execute_reply":"2022-08-02T09:10:54.358195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[2000:12000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:11:07.777080Z","iopub.execute_input":"2022-08-02T09:11:07.777503Z","iopub.status.idle":"2022-08-02T09:11:07.983705Z","shell.execute_reply.started":"2022-08-02T09:11:07.777468Z","shell.execute_reply":"2022-08-02T09:11:07.982643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[5000:15000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:10:28.887319Z","iopub.execute_input":"2022-08-02T09:10:28.887744Z","iopub.status.idle":"2022-08-02T09:10:29.094408Z","shell.execute_reply.started":"2022-08-02T09:10:28.887706Z","shell.execute_reply":"2022-08-02T09:10:29.093229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"100% on recall, but 2000 more false prediction","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:12000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:11:28.857738Z","iopub.execute_input":"2022-08-02T09:11:28.858160Z","iopub.status.idle":"2022-08-02T09:11:29.082388Z","shell.execute_reply.started":"2022-08-02T09:11:28.858124Z","shell.execute_reply":"2022-08-02T09:11:29.081167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"100% on recall, but 3000 more false prediction\n\nstill above 0.97","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:13000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:11:36.370423Z","iopub.execute_input":"2022-08-02T09:11:36.370945Z","iopub.status.idle":"2022-08-02T09:11:36.583630Z","shell.execute_reply.started":"2022-08-02T09:11:36.370899Z","shell.execute_reply":"2022-08-02T09:11:36.582409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"100% on recall, but 4000 more false prediction\n\nDROPS DRAMATICALLY","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'target': [0] * 100000, 'prediction': [0] * 100000})\ndf.loc[:10000, 'target'] = 1\ndf.loc[:14000, 'prediction'] = 1\namex_metric(df[['target']], df[['prediction']])","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:11:44.501617Z","iopub.execute_input":"2022-08-02T09:11:44.502045Z","iopub.status.idle":"2022-08-02T09:11:44.730180Z","shell.execute_reply.started":"2022-08-02T09:11:44.502009Z","shell.execute_reply":"2022-08-02T09:11:44.729100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}