{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U py-boost","metadata":{"execution":{"iopub.status.busy":"2022-05-31T06:28:13.475923Z","iopub.execute_input":"2022-05-31T06:28:13.476816Z","iopub.status.idle":"2022-05-31T06:28:24.804386Z","shell.execute_reply.started":"2022-05-31T06:28:13.47672Z","shell.execute_reply":"2022-05-31T06:28:24.803117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport os\nimport sys\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandas import Series, DataFrame\n\nimport cupy as cp\nimport cudf\nfrom py_boost import GradientBoosting\nfrom py_boost.gpu.losses import Metric\n\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-31T06:28:24.807436Z","iopub.execute_input":"2022-05-31T06:28:24.809244Z","iopub.status.idle":"2022-05-31T06:28:42.28871Z","shell.execute_reply.started":"2022-05-31T06:28:24.809208Z","shell.execute_reply":"2022-05-31T06:28:42.287937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing \n\nThe following script in the hidden cell will read the data with **RAPIDS CuDF** and save to **.feather**. Also it splits the test data into the 4 partitions to prevent OOM during inference. It takes quite a long time (about 30 min) in Kernel probably due to RAM limitations, but locally it takes about 2 mins, so I just use precomputed data. But you may set PREPROCESSING=True if do not want to use the external dataset.","metadata":{}},{"cell_type":"code","source":"script = \"\"\"\nimport os\nimport numpy as np\nimport math\nimport pandas as pd\nimport cudf\nimport subprocess\n\n        \ndef process_fn(path, **kwargs):\n    \n    data = cudf.read_csv(path, dtype=DTYPES, **kwargs)\n    data['customer_ID'] = data['customer_ID'].map(cudf.from_pandas(MAPPER))\n    data['S_2'] = cudf.to_datetime(data['S_2'])\n\n    for col in CATS:\n        if col in FILL_DICT:\n            data[col] += FILL_DICT[col]\n            data[col] = data[col].fillna(0).astype(np.uint8)\n        else:\n            data[col] = data[col].map(cudf.Series(MAP_DICT[col], dtype=np.uint8)).fillna(0)\n            \n    return data\n\n\ndef batch_process_fn(path, batch_size, output, start=0, stop=None):\n    \n    names = pd.read_csv(path, nrows=2).columns.to_list()\n    \n    # iterate over file\n    parts = []\n    skiprows = 1 + start\n    if stop is None:\n        # nrows + 1 \n        stop = int(subprocess.check_output(\n            ['wc', '-l', path]).decode(\"utf-8\").split(' ')[0]) - 1\n        \n    total = stop - start\n    niter = math.ceil(total / batch_size)\n    print(start, stop, total)\n\n    for i in range(niter):\n        batch_size_ = min(batch_size, total - i * batch_size)\n        \n        data = process_fn(path, skiprows=skiprows, nrows=batch_size_, names=names)\n        skiprows += batch_size\n        data = data.to_pandas()\n        parts.append(data)\n        \n        print(data.shape)\n\n    data = pd.concat(parts, axis=0, ignore_index=True)\n    data.to_feather(output)\n    print(data.shape)\n    \n\nif __name__ == '__main__':\n    \n    os.makedirs('./data', exist_ok=True)\n    \n    train_ids = pd.read_csv('../input/amex-default-prediction/train_labels.csv', )\n    test_ids = pd.read_csv('../input/amex-default-prediction/sample_submission.csv', )\n\n    ids = pd.concat([train_ids['customer_ID'], test_ids['customer_ID']], axis=0, \n                   ).sort_values().reset_index()\n    ids['index'] = ids['index'].astype(np.int32)\n\n    MAPPER = ids.set_index('customer_ID')['index']\n\n    train_ids['my_id'] = train_ids['customer_ID'].map(MAPPER).values\n    test_ids['my_id'] = test_ids['customer_ID'].map(MAPPER).values\n\n    train_ids.to_feather('./data/target.feather')\n    test_ids.to_feather('./data/ssub.feather')\n\n    # analyze sample\n    sample = pd.read_csv('../input/amex-default-prediction/train_data.csv', nrows=100)\n    CATS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n    NUMS = sample.columns.drop(['customer_ID', 'S_2'] + CATS).to_list()\n    FEATS = sample.columns.to_list()\n\n    FILL_DICT = {\n        'B_30': 1, 'B_38': 0, 'D_114': 1, 'D_116': 1, 'D_117': 2, 'D_120': 1, 'D_126': 2,  'D_66': 1, 'D_68': 1\n    }\n\n    MAP_DICT = {\n        'D_63': {'CO': 1, 'CR': 2, 'CL': 3, 'XZ': 4, 'XM': 5, 'XL': 6 },\n        'D_64': {'O': 2, 'U': 3, 'R': 4, '-1' : 1, },    \n    }\n\n    DTYPES = sample.dtypes.to_dict()\n\n    for col in DTYPES:\n        if DTYPES[col] == np.float64:\n            DTYPES[col] = np.float32\n            \n    # save feathers\n    batch_process_fn('../input/amex-default-prediction/train_data.csv', 2000000, './data/train.feather')\n    \n    partitions = [\n\n        (0, 2681973), \n        (2681973, 5362891), \n        (5362891, 8045043), \n        (8045043, None)\n\n    ]\n\n    for n, (start, stop) in enumerate(partitions):\n        batch_process_fn(\n            '../input/amex-default-prediction/test_data.csv', 2000000, \n            './data/test_part{0}.feather'.format(n), \n            start=start, stop=stop\n        )\n    \n\"\"\"\nwith open('preprocess_script.py', 'w') as f:\n    f.writelines(script)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-30T23:06:04.804507Z","iopub.execute_input":"2022-05-30T23:06:04.804925Z","iopub.status.idle":"2022-05-30T23:06:04.813556Z","shell.execute_reply.started":"2022-05-30T23:06:04.804877Z","shell.execute_reply":"2022-05-30T23:06:04.812507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPREPROCESSING = False\n\nif PREPROCESSING:\n    !{sys.executable} preprocess_script.py\n    DATA_DIR = './data'\nelse:\n    DATA_DIR = '../input/amexsavedasfeather/'","metadata":{"execution":{"iopub.status.busy":"2022-05-31T06:29:33.36561Z","iopub.execute_input":"2022-05-31T06:29:33.36614Z","iopub.status.idle":"2022-05-31T06:29:33.37526Z","shell.execute_reply.started":"2022-05-31T06:29:33.366092Z","shell.execute_reply":"2022-05-31T06:29:33.374409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature extraction\n\nHere we create aggregated dataset. To simplify the process I used the features from [this Kernel](https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created) proposed by [huseyincotel](https://www.kaggle.com/huseyincot), just speed up execution with **CuDF**","metadata":{}},{"cell_type":"code","source":"def feature_extraction(path):\n    \"\"\"\n    Simple features\n    \"\"\"\n    data = cudf.read_feather(path)\n    \n    CATS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n    NUMS = data.columns.drop(['customer_ID', 'S_2'] + CATS).to_list()\n    \n    agg_nums = data.groupby('customer_ID')[NUMS].agg(['mean', 'std', 'min', 'max', 'last']).astype(np.float32)\n    agg_nums.columns = ['_'.join(x) for x in agg_nums.columns]\n    \n    agg_cats = data.groupby(\"customer_ID\")[CATS].agg(['count', 'last', 'nunique']).astype(np.float32)\n    agg_cats.columns = ['_'.join(x) for x in agg_cats.columns]\n    \n    data = cudf.concat([agg_nums, agg_cats, ], axis=1).to_pandas()\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:56:49.659345Z","iopub.execute_input":"2022-05-30T21:56:49.660189Z","iopub.status.idle":"2022-05-30T21:56:49.671387Z","shell.execute_reply.started":"2022-05-30T21:56:49.660151Z","shell.execute_reply":"2022-05-30T21:56:49.670666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# DataFrames\ntrain = feature_extraction(os.path.join(DATA_DIR, 'train.feather'))\nlabels = pd.read_feather(os.path.join(DATA_DIR, 'target.feather')).set_index('my_id')\n\n# features arrays\nX = train.values\ny = labels['target'].loc[train.index].values","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:56:49.672948Z","iopub.execute_input":"2022-05-30T21:56:49.673294Z","iopub.status.idle":"2022-05-30T21:57:39.492044Z","shell.execute_reply.started":"2022-05-30T21:56:49.67326Z","shell.execute_reply":"2022-05-30T21:57:39.491182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metric comparison\n\nProposed by host metric implementation based on **pandas** looks not very efficient. Lets take some random (not nullable) feature as prediction and evaluate the time","metadata":{}},{"cell_type":"code","source":"# lets create simple dummy predicion\ndummy_pred = X[:, 10]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:39.493365Z","iopub.execute_input":"2022-05-30T21:57:39.493858Z","iopub.status.idle":"2022-05-30T21:57:39.498196Z","shell.execute_reply.started":"2022-05-30T21:57:39.493822Z","shell.execute_reply":"2022-05-30T21:57:39.497426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_official(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:39.499841Z","iopub.execute_input":"2022-05-30T21:57:39.500826Z","iopub.status.idle":"2022-05-30T21:57:39.514799Z","shell.execute_reply.started":"2022-05-30T21:57:39.500785Z","shell.execute_reply":"2022-05-30T21:57:39.514013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# proposed implementation takes DataFrame as input, so lets do so\nydf = DataFrame({'target': y})\nxdf = DataFrame({'prediction': dummy_pred})","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:39.515994Z","iopub.execute_input":"2022-05-30T21:57:39.516806Z","iopub.status.idle":"2022-05-30T21:57:39.528587Z","shell.execute_reply.started":"2022-05-30T21:57:39.516768Z","shell.execute_reply":"2022-05-30T21:57:39.527729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\nval = amex_metric_official(ydf, xdf)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:39.532896Z","iopub.execute_input":"2022-05-30T21:57:39.533557Z","iopub.status.idle":"2022-05-30T21:57:47.120203Z","shell.execute_reply.started":"2022-05-30T21:57:39.533519Z","shell.execute_reply":"2022-05-30T21:57:47.119224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric_official(ydf, xdf)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:47.121464Z","iopub.execute_input":"2022-05-30T21:57:47.122265Z","iopub.status.idle":"2022-05-30T21:57:47.983658Z","shell.execute_reply.started":"2022-05-30T21:57:47.122225Z","shell.execute_reply":"2022-05-30T21:57:47.982793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Takes about 1 second. Not too long to evaluate the model quality, but almost impossible to use as early stopping detector for tree ensembles\n\n### Numpy implementation\n\nImplementation based. on **NumPy** looks much more efficient. It is proposed by [Konstantin Yakovlev](https://www.kaggle.com/kyakovlev) in this post [this post](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534). Lets see how fast it is.","metadata":{}},{"cell_type":"code","source":"def amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:47.985039Z","iopub.execute_input":"2022-05-30T21:57:47.985406Z","iopub.status.idle":"2022-05-30T21:57:47.994771Z","shell.execute_reply.started":"2022-05-30T21:57:47.98537Z","shell.execute_reply":"2022-05-30T21:57:47.993894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\nval = amex_metric_numpy(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:57:47.996204Z","iopub.execute_input":"2022-05-30T21:57:47.996847Z","iopub.status.idle":"2022-05-30T21:58:02.812939Z","shell.execute_reply.started":"2022-05-30T21:57:47.996767Z","shell.execute_reply":"2022-05-30T21:58:02.812072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric_numpy(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:02.814073Z","iopub.execute_input":"2022-05-30T21:58:02.814882Z","iopub.status.idle":"2022-05-30T21:58:02.995742Z","shell.execute_reply.started":"2022-05-30T21:58:02.814843Z","shell.execute_reply":"2022-05-30T21:58:02.994864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Good job! Almost x5 speed up. But can we make it faster? :)","metadata":{}},{"cell_type":"code","source":"def check_input(arr):\n    \"\"\"\n    Check the input\n    \"\"\"\n    if type(arr) is pd.DataFrame:\n        arr = arr[arr.columns[0]]\n        \n    if type(arr) is pd.Series:\n        arr = arr.values\n        \n    if len(arr.shape) > 1:\n        arr = arr[:, 0]\n        \n    return arr\n\n\ndef gini(cs_0, cs_1, sum_0, sum_1):\n    \"\"\"\n    Gini part\n    \"\"\"\n    auc_ = (cs_0 - sum_0 / 2) * sum_1\n    tot = cs_0[-1] * cs_1[-1]\n\n    return 2 * float(auc_.sum() / tot) - 1\n\n\ndef recall_at4(cs_0, cs_1, sum_1):\n    \"\"\"\n    Recall part\n    \"\"\"\n    cs_tot = cs_0 + cs_1\n    th = cs_tot[-1] * 0.96\n    \n    return float(sum_1[cs_tot >= th].sum() / cs_1[-1])\n    \n    \ndef amex_score(y_true, y_pred):\n    \"\"\"\n    Faster NumPy metric implementation\n    \"\"\"\n    y_true = check_input(y_true)\n    y_pred = check_input(y_pred)\n\n    sum_1 = y_true[y_pred.argsort()]\n    sum_0 = (1 - sum_1) \n    sum_0 *= 20\n    \n    cs_0, cs_1 = np.cumsum(sum_0, dtype=np.float64), np.cumsum(sum_1, dtype=np.float64)\n    \n    g = gini(cs_0, cs_1, sum_0, sum_1)\n    d = recall_at4(cs_0, cs_1, sum_1)\n    \n    return (g + d) / 2","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:02.996875Z","iopub.execute_input":"2022-05-30T21:58:02.997696Z","iopub.status.idle":"2022-05-30T21:58:03.009654Z","shell.execute_reply.started":"2022-05-30T21:58:02.99765Z","shell.execute_reply":"2022-05-30T21:58:03.008746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\nval = amex_score(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:03.011232Z","iopub.execute_input":"2022-05-30T21:58:03.011792Z","iopub.status.idle":"2022-05-30T21:58:07.752211Z","shell.execute_reply.started":"2022-05-30T21:58:03.011756Z","shell.execute_reply":"2022-05-30T21:58:07.751331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_score(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:07.753375Z","iopub.execute_input":"2022-05-30T21:58:07.75421Z","iopub.status.idle":"2022-05-30T21:58:07.819351Z","shell.execute_reply.started":"2022-05-30T21:58:07.75418Z","shell.execute_reply":"2022-05-30T21:58:07.81839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### This NumPy implementation is about x3 faster. But remember, this is GPU Kernel, so let's use it : )","metadata":{}},{"cell_type":"code","source":"def amex_score_gpu(y_true, y_pred):\n    \"\"\"\n    CuPy based AMEX metric\n    \"\"\"\n    y_true = cp.asarray(check_input(y_true))\n    y_pred = cp.asarray(check_input(y_pred))\n    \n    unique = cp.unique(y_pred)\n    rank = cp.searchsorted(unique, y_pred)\n    \n    sum_1 = cp.zeros_like(unique, dtype=cp.float64)\n    sum_1.scatter_add(rank, y_true)\n    \n    sum_0 = cp.zeros_like(unique, dtype=cp.float64)\n    sum_0.scatter_add(rank, 1 - y_true)\n    sum_0 *= 20\n    \n    cs_0, cs_1 = sum_0.cumsum(), sum_1.cumsum()\n    \n    g = gini(cs_0, cs_1, sum_0, sum_1)\n    d = recall_at4(cs_0, cs_1, sum_1)\n    \n    return (g + d) / 2","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:07.820939Z","iopub.execute_input":"2022-05-30T21:58:07.821345Z","iopub.status.idle":"2022-05-30T21:58:07.828607Z","shell.execute_reply.started":"2022-05-30T21:58:07.821307Z","shell.execute_reply":"2022-05-30T21:58:07.827537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\nval = amex_score_gpu(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:07.830213Z","iopub.execute_input":"2022-05-30T21:58:07.83087Z","iopub.status.idle":"2022-05-30T21:58:14.110831Z","shell.execute_reply.started":"2022-05-30T21:58:07.830831Z","shell.execute_reply":"2022-05-30T21:58:14.10998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_score_gpu(y, dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:14.112229Z","iopub.execute_input":"2022-05-30T21:58:14.11276Z","iopub.status.idle":"2022-05-30T21:58:14.134144Z","shell.execute_reply.started":"2022-05-30T21:58:14.112721Z","shell.execute_reply":"2022-05-30T21:58:14.133379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### And x5 more using GPU. Some time is spent for CPU->GPU data transfer, so if target and predictions already will be on the device, it will be a liite bit faster","metadata":{}},{"cell_type":"code","source":"ygpu = cp.asarray(y)\nxgpu = cp.asarray(dummy_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:14.135557Z","iopub.execute_input":"2022-05-30T21:58:14.13596Z","iopub.status.idle":"2022-05-30T21:58:14.142671Z","shell.execute_reply.started":"2022-05-30T21:58:14.135907Z","shell.execute_reply":"2022-05-30T21:58:14.141615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\nval = amex_score_gpu(ygpu, xgpu)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:14.144961Z","iopub.execute_input":"2022-05-30T21:58:14.145385Z","iopub.status.idle":"2022-05-30T21:58:21.093681Z","shell.execute_reply.started":"2022-05-30T21:58:14.145351Z","shell.execute_reply":"2022-05-30T21:58:21.091986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So why do we need so fast metric? The answer is - to perform early stopping in Gradient Boosting. And here is what I want to show\n\n### Py-Boost\n\n**Py-Boost** is GBDT implementation written only on Python. Despite this fact, it is fast because it is based on Python GPU libraries (**CuPy** mostly). I made it few month ago mainly for the research purposes, that is focused on multioutput training (multilabel/multiclass tasks), where **py-boost** is very efficient. But we can also train a simple binary task too. \n\nTo learn more, visit our [Github repo](https://github.com/sb-ai-lab/Py-Boost). Here you will find some more usage tutorials. If you like this tool, you also can star us :)\n\nDuring **Py-Boost** traing data is represented as GPU arrays, so we can loosely pass  **amex_score_gpu** function for the fast evaluation with no overhead on data transfer, just need to write the wrapper class","metadata":{}},{"cell_type":"code","source":"class CustomAMEXMetric(Metric):\n    \"\"\"Custom AMEX Metric for Py-Boost\"\"\"\n    \n    def compare(self, v0 ,v1):\n        \"\"\"\n        It should return True if v0 (new value) metric value is better than v1 (old value), False othewise\n        \"\"\"\n        return v0 > v1\n    \n    def __call__(self, y_true, y_pred, sample_weight=None):\n        \"\"\"\n        We also update __call__ method with AMEX score\n        \"\"\"\n        return amex_score_gpu(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T21:58:21.095209Z","iopub.execute_input":"2022-05-30T21:58:21.095619Z","iopub.status.idle":"2022-05-30T21:58:21.101824Z","shell.execute_reply.started":"2022-05-30T21:58:21.095576Z","shell.execute_reply":"2022-05-30T21:58:21.100534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Other **py-boost** hyperparameters and interface are very common for all GBDT users. Here is the example of Cross-Validation loop with custom AMEX metric:","metadata":{}},{"cell_type":"code","source":"params = {\n\n    'loss': 'bce',\n    'metric': CustomAMEXMetric(),\n    'ntrees': 10000,\n    'lr': 0.01,\n    'min_gain_to_split': 0,\n    'lambda_l2': 10,\n    'max_bin': 256,\n    'max_depth': 6,\n    'min_data_in_leaf': 10,\n    'colsample': 0.9,\n    'subsample': 0.6,\n    'quant_sample': 1000000,\n    'es': 300,\n    'verbose': 100,\n    \n}\n\ndef cv_loop(params, X, y, score=amex_score):\n    \"\"\"\n    CrossValidation loop for Py-Boost training\n    \"\"\"\n    scores = []\n    models = []\n    indices = np.arange(X.shape[0])\n    oof_pred = np.zeros((X.shape[0], ), dtype=np.float32)\n    cv = StratifiedKFold(5, shuffle=True, random_state=42)\n    \n    for n, (f0, f1) in enumerate(cv.split(y, y)):\n        \n        X_test, y_test = X[f1], y[f1]\n        \n        model = GradientBoosting(**params)\n        model.fit(X[f0], y[f0], eval_sets=[{'X': X_test, 'y': y_test}])\n        \n        models.append(model)\n        oof_pred[f1] = model.predict(X_test, batch_size=1000000)[:, 0]\n        scores.append(\n            score(y_test, oof_pred[f1])\n        )\n        \n        print('Fold {0}: score {1}'.format(n, scores[-1]))\n        \n    return oof_pred, np.array(scores), models","metadata":{"execution":{"iopub.status.busy":"2022-05-30T22:12:16.649949Z","iopub.execute_input":"2022-05-30T22:12:16.650769Z","iopub.status.idle":"2022-05-30T22:12:16.662044Z","shell.execute_reply.started":"2022-05-30T22:12:16.650731Z","shell.execute_reply":"2022-05-30T22:12:16.661116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\noof_pred, scores, models = cv_loop(params, X, y)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T22:12:16.89883Z","iopub.execute_input":"2022-05-30T22:12:16.899703Z","iopub.status.idle":"2022-05-30T22:41:52.83864Z","shell.execute_reply.started":"2022-05-30T22:12:16.899667Z","shell.execute_reply":"2022-05-30T22:41:52.836515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_score(y, oof_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T22:44:04.274121Z","iopub.execute_input":"2022-05-30T22:44:04.275025Z","iopub.status.idle":"2022-05-30T22:44:04.341276Z","shell.execute_reply.started":"2022-05-30T22:44:04.274989Z","shell.execute_reply":"2022-05-30T22:44:04.34031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature importance\n\nAs usual, you can visualize feature importances","metadata":{}},{"cell_type":"code","source":"Series(models[0].get_feature_importance(), index=train.columns).sort_values(\n    ascending=False)[:20][::-1].plot(kind='barh')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T22:47:07.22956Z","iopub.execute_input":"2022-05-30T22:47:07.229939Z","iopub.status.idle":"2022-05-30T22:47:07.796186Z","shell.execute_reply.started":"2022-05-30T22:47:07.229889Z","shell.execute_reply":"2022-05-30T22:47:07.795393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, X","metadata":{"execution":{"iopub.status.busy":"2022-05-30T22:49:38.403548Z","iopub.execute_input":"2022-05-30T22:49:38.404284Z","iopub.status.idle":"2022-05-30T22:49:38.408681Z","shell.execute_reply.started":"2022-05-30T22:49:38.404243Z","shell.execute_reply":"2022-05-30T22:49:38.407616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference\n\nInference code is also quite simple","metadata":{}},{"cell_type":"code","source":"def inference(flist, models):\n    \"\"\"\n    Inference separated by partitions\n    \"\"\"\n    pred_parts = []\n\n    for path in flist:\n        test = feature_extraction(path)\n        X_test, index = test.values, test.index\n\n        del test\n\n        pred = np.zeros((X_test.shape[0], ), dtype=np.float32)\n\n        for model in models:\n            pred += model.predict(X_test, batch_size=5000000)[:, 0]\n\n        pred /= len(models)\n        pred_parts.append(Series(pred, index=index))\n\n        del X_test\n        print('{0} finished'.format(path))    \n        \n    return pd.concat(pred_parts, axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprediction = inference(\n    [os.path.join(DATA_DIR, 'test_part{0}.feather'.format(x)) for x in range(4)],\n    models\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The last thing to do is generating submission","metadata":{}},{"cell_type":"code","source":"ssub = pd.read_feather(os.path.join(DATA_DIR, 'ssub.feather')).set_index('my_id')\nssub['prediction'] = prediction","metadata":{"execution":{"iopub.status.busy":"2022-05-30T23:01:11.281977Z","iopub.execute_input":"2022-05-30T23:01:11.28285Z","iopub.status.idle":"2022-05-30T23:01:13.141771Z","shell.execute_reply.started":"2022-05-30T23:01:11.282809Z","shell.execute_reply":"2022-05-30T23:01:13.140961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssub[['customer_ID', 'prediction']]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T23:06:51.365384Z","iopub.execute_input":"2022-05-30T23:06:51.365777Z","iopub.status.idle":"2022-05-30T23:06:51.405013Z","shell.execute_reply.started":"2022-05-30T23:06:51.365743Z","shell.execute_reply":"2022-05-30T23:06:51.404113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssub[['customer_ID', 'prediction']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T23:02:34.156539Z","iopub.execute_input":"2022-05-30T23:02:34.156961Z","iopub.status.idle":"2022-05-30T23:02:34.165477Z","shell.execute_reply.started":"2022-05-30T23:02:34.156903Z","shell.execute_reply":"2022-05-30T23:02:34.164355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Hope this kernel was helpful. Thanks for your attention! And Good luck :)","metadata":{}}]}