{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Sebastian Smith\n# Based on notebook \"AMEX Prediction - Starter\" at https://www.kaggle.com/code/mirfanazam/amex-prediction-starter\n# You must install all necessary libraries before running the program to run it on your PC.\n\n# Setup\nimport vaex\nvaex.multithreading.thread_count_default = 8\nimport vaex.ml\n\nimport pandas as pd\nimport numpy  as np \n\nimport os\nimport gc\nimport psutil\nimport glob\n\nimport lightgbm as lgb\nfrom lightgbm import log_evaluation\nfrom sklearn.model_selection import train_test_split\n\n# Utility Functions\n\ndef remove_output_files(file_pattern):\n    fileList = glob.glob(file_pattern)\n    for filePath in fileList:\n        try:\n            os.remove(filePath)\n        except:\n            print(\"Error while deleting file : \", filePath)\n\ndef fill_and_convert_floats(ddf):\n    for c in ddf.columns:\n        if ddf[c].dtype == 'float64':\n            ddf[c] = ddf[c].fillna(0.0).astype('float32')\n    return ddf\n\ndef encode_cat_features(df):\n    cat_features = ['D_63','D_64']\n    label_encoder = vaex.ml.LabelEncoder(features=cat_features)\n    df = label_encoder.fit_transform(df)\n    df.drop(cat_features, inplace=True)\n    df.rename('label_encoded_D_63','D_63')\n    df.rename('label_encoded_D_64','D_64')\n    df['D_64'] = df['D_64'].astype('float32')\n    df['D_63'] = df['D_63'].astype('float32')\n    df['B_31'] = df['B_31'].astype('float32')\n    return df\n\ndef get_last_statement(df):\n    return df.groupby(['customer_ID']).agg({col: vaex.agg.last(col) for col in df.get_column_names() if col not in [\"customer_ID\"]})\n\ndef get_last_statement_ex(df_test):\n    delinquency_features = [col for col in df_test if col.startswith('D_')] \n    df = df_test.groupby(['customer_ID']).agg({col: vaex.agg.last(col) for col in df_test.get_column_names() if col not in delinquency_features + [\"customer_ID\"]})\n    df.export_hdf5('./last-statement-p1.hdf5')\n    del df\n    gc.collect()\n    delinquency_features = ['S_2'] + [col for col in df_test if col.startswith('D_') and len(col) == 4] \n    delinquency_features2 = ['S_2'] + [col for col in df_test if col.startswith('D_') and len(col) == 5]\n    df_2 = df_test.groupby(['customer_ID']).agg({col: vaex.agg.last(col) for col in df_test.get_column_names() if col in delinquency_features})\n    df_2.export_hdf5('./last-statement-p2.hdf5')\n    del df_2\n    gc.collect()\n    df_3 = df_test.groupby(['customer_ID']).agg({col: vaex.agg.last(col) for col in df_test.get_column_names() if col in delinquency_features2})\n    df_3.export_hdf5('./last-statement-p3.hdf5')\n    del df_3\n    gc.collect()\n    last_statement_p1 = vaex.open('./last-statement-p1.hdf5')\n    last_statement_p2 = vaex.open('./last-statement-p2.hdf5')\n    last_statement_p3 = vaex.open('./last-statement-p3.hdf5')\n    last_statement_p1 = last_statement_p1.drop('S_2')\n    last_statement_p2 = last_statement_p2.drop('S_2')\n    last_statement_p3 = last_statement_p3.drop('S_2')\n    gc.collect()\n    last_statement_p1 = last_statement_p1.join(last_statement_p2, how=\"inner\", on='customer_ID')\n    df_last_statement = last_statement_p1.join(last_statement_p3, how=\"inner\", on='customer_ID')\n    del last_statement_p1\n    del last_statement_p2\n    del last_statement_p3\n    gc.collect()\n    statement_path = './last-statement-p*.hdf5'\n    remove_output_files(statement_path)\n    return df_last_statement\n\ndef process_data(data):\n    for i, df in enumerate(vaex.from_csv(f'{data}.csv', chunk_size=500_000)):\n        print(\"process_data i: \"+str(i))\n        df['S_2'] = df['S_2'].str.replace('-','').astype('float32')\n        df = fill_and_convert_floats(df)\n        df = encode_cat_features(df)\n        df = get_last_statement(df)\n        export_path = f'./{data}_{i:02}.hdf5'    \n        df.export_hdf5(export_path)\n        del df\n        gc.collect()\n    import_path = f'./{data}_*.hdf5'\n    df = vaex.open(import_path)\n    df.export_hdf5(f'./{data}.hdf5')\n    del df\n    gc.collect()\n    remove_output_files(import_path)\n\ndef process_data_level2(df, data, flag):\n    if flag == 1:\n        df = get_last_statement_ex(df)     \n    else:\n        df = get_last_statement(df)\n        df.drop('S_2', inplace=True)   \n    label_encoder = vaex.ml.LabelEncoder(features=['customer_ID'])\n    df = label_encoder.fit_transform(df)\n    df_customer_map = df[['label_encoded_customer_ID', 'customer_ID']]\n    df.drop('customer_ID', inplace=True)\n    df.rename('label_encoded_customer_ID','customer_ID')\n    df_customer_map.export_hdf5(f'./{data}_customer_map.hdf5')\n    df.export_hdf5(f'./{data}v2.hdf5')\n    del df\n    del df_customer_map\n    gc.collect()\n\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n    return 0.5 * (g + d)\n\ndef amex_metric_np(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric_np(y_true, y_pred), True\n\nprocess_data('train_data') # Level 1 preprocessing - Reduce train and test datasets\nprocess_data('test_data')\n\ndf_train = vaex.open('./train_data.hdf5') # Uses train and test data from the output folder.\ndf_test = vaex.open('./test_data.hdf5')\n\n# Preprocessing - Level 2\n\nprocess_data_level2(df_train, 'train_data', 0) # Generates level 2 data.\nprocess_data_level2(df_test, 'test_data', 1)\n\ndf_train = vaex.open('./train_datav2.hdf5') # Uses train and test data from the output folder.\ndf_test = vaex.open('./test_datav2.hdf5')\ndf_train_map = vaex.open('./train_data_customer_map.hdf5')\ndf_test_map = vaex.open('./test_data_customer_map.hdf5')\n\n# Prepares data to train the model\n\ndf_train_labels = vaex.open('./train_labels.csv')\ndf_train_labels = df_train_labels.join(df_train_map, how=\"inner\", on=\"customer_ID\")\nall_features = [col for col in df_train]\ndf_customer = df_train[all_features]\ndf_customer = df_customer.join(df_train_labels, left_on='customer_ID', right_on='label_encoded_customer_ID', how='inner')\ndf_customer.drop(['label_encoded_customer_ID'], inplace=True)\n\n# Converts Vaex DataFrames to Pandas DataFrames\n\ndf_train = df_train.to_pandas_df()\ndf_test = df_test.to_pandas_df()\ndf_train_map = df_train_map.to_pandas_df()\ndf_test_map = df_test_map.to_pandas_df()\ndf_train_labels = df_train_labels.to_pandas_df()\ndf_customer = df_customer.to_pandas_df()\n\n# Trains the model\n\ny = df_customer.pop('target')\nmodel_features = [col for col in df_customer]\nX = df_customer[model_features]\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\ndtrain = lgb.Dataset(\n    data=X_train,\n    label=y_train\n)\n\ndvalid = lgb.Dataset(\n    data=X_test,\n    label=y_test,\n    reference=dtrain\n)\n\nprint(len(X_test))\nprint(len(y_test))\n\ncategorical_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndf_customer['D_117'] = df_customer['D_117'] + 1\ndf_customer['D_126'] = df_customer['D_126'] + 1\n\nlgb_params={\n    'objective': \"binary\",\n    'num_iterations': 1200,\n    'learning_rate': 0.03,\n    'reg_lambda': 50,\n    'min_child_samples': 2400,\n    'num_leaves': 220,\n    'colsample_bytree': 0.19,\n    'random_state': 1,\n    'verbose': -1\n}\n\nmodel = lgb.train(\n    params=lgb_params,\n    train_set=dtrain,\n    valid_sets=[dvalid],\n    feval=lgb_amex_metric,\n    callbacks=[log_evaluation(100)]\n)\n\ny_pred = model.predict(X_test)\n\n# Makes the default predictions.\n\nmodel_features = [col for col in df_customer]\ndf_customer_test = df_test[model_features]\ndf_customer_test_pred = model.predict(df_customer_test)\n\n# Makes the submission CSV file.\n\ndf_customer_test = pd.merge(df_customer_test, df_test_map, how=\"inner\", left_on=\"customer_ID\", right_on=\"label_encoded_customer_ID\")\ndf_customer_test = df_customer_test.drop(columns={'customer_ID_x','label_encoded_customer_ID'})\ndf_customer_test_pred = pd.DataFrame(df_customer_test_pred.tolist())\ndf_customer_test['prediction'] = df_customer_test_pred\ndf_customer_test = df_customer_test.rename(columns={\"customer_ID_y\": \"customer_ID\"})\nfinal_prediction = df_customer_test[[\"customer_ID\", \"prediction\"]]\nfinal_prediction.to_csv(\"./submission.csv\",index=False)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T05:49:46.965074Z","iopub.execute_input":"2022-07-22T05:49:46.966013Z","iopub.status.idle":"2022-07-22T05:49:51.303902Z","shell.execute_reply.started":"2022-07-22T05:49:46.965845Z","shell.execute_reply":"2022-07-22T05:49:51.302017Z"},"trusted":true},"execution_count":null,"outputs":[]}]}