{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":162403520,"sourceType":"kernelVersion"},{"sourceId":162424208,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":1479.820182,"end_time":"2024-02-10T05:23:05.644646","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-10T04:58:25.824464","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.00736,"end_time":"2024-02-10T04:58:28.581611","exception":false,"start_time":"2024-02-10T04:58:28.574251","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Related notebooks**\n\nUtility script notebook (with additional functions, aggregators):\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-data\n\nInference notebook:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-inference","metadata":{"papermill":{"duration":0.006566,"end_time":"2024-02-10T04:58:28.594993","exception":false,"start_time":"2024-02-10T04:58:28.588427","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.006713,"end_time":"2024-02-10T04:58:28.609783","exception":false,"start_time":"2024-02-10T04:58:28.603070","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --no-index -Uq --find-links=/kaggle/input/lightautoml-038-dependencies lightautoml==0.3.8","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-10T19:29:10.955574Z","iopub.execute_input":"2024-02-10T19:29:10.956261Z","iopub.status.idle":"2024-02-10T19:29:25.502203Z","shell.execute_reply.started":"2024-02-10T19:29:10.956226Z","shell.execute_reply":"2024-02-10T19:29:25.501147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport lightgbm as lgb\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.303403,"end_time":"2024-02-10T04:58:34.920027","exception":false,"start_time":"2024-02-10T04:58:28.616624","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-10T19:29:25.504350Z","iopub.execute_input":"2024-02-10T19:29:25.504674Z","iopub.status.idle":"2024-02-10T19:29:35.298721Z","shell.execute_reply.started":"2024-02-10T19:29:25.504643Z","shell.execute_reply":"2024-02-10T19:29:35.297833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightautoml.automl.presets.tabular_presets import TabularAutoML\nfrom lightautoml.tasks import Task\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:29:35.299935Z","iopub.execute_input":"2024-02-10T19:29:35.300659Z","iopub.status.idle":"2024-02-10T19:30:02.502440Z","shell.execute_reply.started":"2024-02-10T19:29:35.300615Z","shell.execute_reply":"2024-02-10T19:30:02.501535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data collection","metadata":{"papermill":{"duration":0.006776,"end_time":"2024-02-10T04:58:34.934015","exception":false,"start_time":"2024-02-10T04:58:34.927239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:30:02.504760Z","iopub.execute_input":"2024-02-10T19:30:02.505390Z","iopub.status.idle":"2024-02-10T19:30:02.519098Z","shell.execute_reply.started":"2024-02-10T19:30:02.505362Z","shell.execute_reply":"2024-02-10T19:30:02.518062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import utility script notebook with data collection functions. See notebook for details: [Home Credit: baseline - DATA](https://www.kaggle.com/code/andreynesterov/home-credit-baseline-data)","metadata":{"papermill":{"duration":0.006661,"end_time":"2024-02-10T04:58:34.947624","exception":false,"start_time":"2024-02-10T04:58:34.940963","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import home_credit_baseline_data as data_nb","metadata":{"papermill":{"duration":0.025934,"end_time":"2024-02-10T04:58:34.980320","exception":false,"start_time":"2024-02-10T04:58:34.954386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-10T19:30:02.520106Z","iopub.execute_input":"2024-02-10T19:30:02.520352Z","iopub.status.idle":"2024-02-10T19:30:02.545672Z","shell.execute_reply.started":"2024-02-10T19:30:02.520330Z","shell.execute_reply":"2024-02-10T19:30:02.544980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet('/kaggle/usr/lib/home_credit_baseline_data/train_full.parquet')\ncat_cols = list(train_df.select_dtypes(\"category\").columns)\ndisplay(train_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":142.824103,"end_time":"2024-02-10T05:00:57.811638","exception":false,"start_time":"2024-02-10T04:58:34.987535","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-10T19:30:02.546598Z","iopub.execute_input":"2024-02-10T19:30:02.546864Z","iopub.status.idle":"2024-02-10T19:30:13.048546Z","shell.execute_reply.started":"2024-02-10T19:30:02.546841Z","shell.execute_reply":"2024-02-10T19:30:13.047592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(cat_cols)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.027724,"end_time":"2024-02-10T05:00:57.856133","exception":false,"start_time":"2024-02-10T05:00:57.828409","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-02-10T19:30:13.049752Z","iopub.execute_input":"2024-02-10T19:30:13.050030Z","iopub.status.idle":"2024-02-10T19:30:13.058020Z","shell.execute_reply.started":"2024-02-10T19:30:13.050006Z","shell.execute_reply":"2024-02-10T19:30:13.056999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(data_nb.CFG.test_dir, cat_cols=cat_cols, mode=\"test\", train_cols=train_df.columns)\ndisplay(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.993087,"end_time":"2024-02-10T05:00:58.866216","exception":false,"start_time":"2024-02-10T05:00:57.873129","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-10T19:30:13.059354Z","iopub.execute_input":"2024-02-10T19:30:13.059729Z","iopub.status.idle":"2024-02-10T19:30:14.366391Z","shell.execute_reply.started":"2024-02-10T19:30:13.059694Z","shell.execute_reply":"2024-02-10T19:30:14.365448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop columns","metadata":{"papermill":{"duration":0.025427,"end_time":"2024-02-10T05:00:58.917438","exception":false,"start_time":"2024-02-10T05:00:58.892011","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_cols = []\ndrop_cols_startwith = [\"mean_\", \"first_\"]\nfor name_prefix in drop_cols_startwith:\n    cols_names = train_df.columns[train_df.columns.str.startswith(name_prefix)]\n    drop_cols += cols_names.to_list()\ndisplay(drop_cols)\ndisplay(len(drop_cols))","metadata":{"papermill":{"duration":0.032795,"end_time":"2024-02-10T05:00:58.975800","exception":false,"start_time":"2024-02-10T05:00:58.943005","status":"completed"},"scrolled":true,"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-10T19:30:14.367465Z","iopub.execute_input":"2024-02-10T19:30:14.367783Z","iopub.status.idle":"2024-02-10T19:30:14.380448Z","shell.execute_reply.started":"2024-02-10T19:30:14.367756Z","shell.execute_reply":"2024-02-10T19:30:14.379377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(columns=drop_cols, inplace=True)\ntest_df.drop(columns=drop_cols, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:30:14.385617Z","iopub.execute_input":"2024-02-10T19:30:14.385933Z","iopub.status.idle":"2024-02-10T19:30:15.453146Z","shell.execute_reply.started":"2024-02-10T19:30:14.385909Z","shell.execute_reply":"2024-02-10T19:30:15.452272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:30:15.454314Z","iopub.execute_input":"2024-02-10T19:30:15.454616Z","iopub.status.idle":"2024-02-10T19:30:15.461807Z","shell.execute_reply.started":"2024-02-10T19:30:15.454589Z","shell.execute_reply":"2024-02-10T19:30:15.460714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = data_nb.reduce_mem_usage(train_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:30:15.463015Z","iopub.execute_input":"2024-02-10T19:30:15.463349Z","iopub.status.idle":"2024-02-10T19:32:05.136608Z","shell.execute_reply.started":"2024-02-10T19:30:15.463313Z","shell.execute_reply":"2024-02-10T19:32:05.135598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.reduce_mem_usage(test_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:32:05.138293Z","iopub.execute_input":"2024-02-10T19:32:05.138599Z","iopub.status.idle":"2024-02-10T19:32:05.351245Z","shell.execute_reply.started":"2024-02-10T19:32:05.138572Z","shell.execute_reply":"2024-02-10T19:32:05.350266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:32:05.352666Z","iopub.execute_input":"2024-02-10T19:32:05.353384Z","iopub.status.idle":"2024-02-10T19:32:05.628598Z","shell.execute_reply.started":"2024-02-10T19:32:05.353347Z","shell.execute_reply":"2024-02-10T19:32:05.627345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.025165,"end_time":"2024-02-10T05:00:59.026486","exception":false,"start_time":"2024-02-10T05:00:59.001321","status":"completed"},"tags":[]}},{"cell_type":"code","source":"N_THREADS = 4\nN_FOLDS = 5\nRANDOM_STATE = 2024\nTIMEOUT = 10000\nADVANCED_ROLES = False\nUSE_QNT = False\nUSE_PLR = False\nTRAIN_BS = 128\nEPOCHS = 5\nTARGET_NAME = 'target'\n\nnp.random.seed(RANDOM_STATE)\ntorch.set_num_threads(N_THREADS)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:32:05.629913Z","iopub.execute_input":"2024-02-10T19:32:05.630209Z","iopub.status.idle":"2024-02-10T19:32:05.646663Z","shell.execute_reply.started":"2024-02-10T19:32:05.630183Z","shell.execute_reply":"2024-02-10T19:32:05.645886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_class(x, task, reader):\n    if task.name == 'multiclass':\n        return reader[x]\n    else:\n        return x\n\nmapped = np.vectorize(map_class)\n\ndef score(task, y_true, y_pred):\n    if task.name == 'binary':\n        return roc_auc_score(y_true, y_pred)\n    elif task.name == 'multiclass':\n        return log_loss(y_true, y_pred)\n    elif task.name == 'reg' or task.name == 'multi:reg':\n        return mean_absolute_error(y_true, y_pred)\n    else:\n        raise 'Task is not correct.'\n        \ndef take_pred_from_task(pred, task):\n    if task.name == 'binary' or task.name == 'reg':\n        return pred[:, 0]\n    elif task.name == 'multiclass' or task.name == 'multi:reg':\n        return pred\n    else:\n        raise 'Task is not correct.'\n        \ndef use_plr(USE_PLR):\n    if USE_PLR:\n        return \"plr\"\n    else:\n        return \"cont\"","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:32:05.648372Z","iopub.execute_input":"2024-02-10T19:32:05.649209Z","iopub.status.idle":"2024-02-10T19:32:05.658403Z","shell.execute_reply.started":"2024-02-10T19:32:05.649174Z","shell.execute_reply":"2024-02-10T19:32:05.657654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task = Task(\n    'binary', \n    loss = 'logloss', \n    metric = 'auc'\n)\nroles = {\n    'target': TARGET_NAME,\n    'group': \"WEEK_NUM\",\n    'drop': ['case_id', \"WEEK_NUM\"],\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:32:05.660042Z","iopub.execute_input":"2024-02-10T19:32:05.660775Z","iopub.status.idle":"2024-02-10T19:32:05.678430Z","shell.execute_reply.started":"2024-02-10T19:32:05.660742Z","shell.execute_reply":"2024-02-10T19:32:05.677580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nalgo = 'denselight'\nautoml_model = TabularAutoML(\n    task = task, \n    timeout = TIMEOUT,\n    cpu_limit = N_THREADS,\n    gpu_ids='0',\n    general_params = {\"use_algos\": [[algo]]}, # ['nn', 'mlp', 'dense', 'denselight', 'resnet', 'snn', 'node', 'autoint', 'fttransformer'] or custom torch model\n    nn_params = {\n        \"n_epochs\": EPOCHS, \n        \"bs\": TRAIN_BS, \n        \"num_workers\": 0, \n        \"path_to_save\": None, \n        \"freeze_defaults\": True,\n        \"cont_embedder\": use_plr(USE_PLR),\n    },\n    nn_pipeline_params = {\n        \"use_qnt\": USE_QNT, \n        \"use_te\": False\n    },\n    reader_params = {\n        'n_jobs': N_THREADS, \n        'cv': N_FOLDS, \n        'random_state': RANDOM_STATE, \n        'advanced_roles': ADVANCED_ROLES\n    },\n)\n\noof_pred = automl_model.fit_predict(train_df, roles = roles, verbose = 3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-10T19:32:05.679797Z","iopub.execute_input":"2024-02-10T19:32:05.680474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_pred.data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = score(\n    task,\n    mapped(train_df[TARGET_NAME].values, task, automl_model.reader.class_mapping),\n    take_pred_from_task(oof_pred.data, task)\n)\nprint(\"CV roc_auc_oof: \", oof)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(oof_pred.data, 'denselight_oof_preds.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(automl_model, 'denselight_model.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump((train_df.columns, cat_cols, drop_cols), \"train_cat_columns.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oof_pred\ngc.collect()","metadata":{"papermill":{"duration":0.163109,"end_time":"2024-02-10T05:23:03.097082","exception":false,"start_time":"2024-02-10T05:23:02.933973","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"papermill":{"duration":0.031773,"end_time":"2024-02-10T05:23:03.162540","exception":false,"start_time":"2024-02-10T05:23:03.130767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=100000):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        batch_probs = model.predict(X_batch).data.squeeze()\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"papermill":{"duration":0.041537,"end_time":"2024-02-10T05:23:03.235674","exception":false,"start_time":"2024-02-10T05:23:03.194137","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred = pd.Series(predict_proba_in_batches(automl_model, X_test), index=X_test.index)\ny_pred[:10]","metadata":{"papermill":{"duration":0.706127,"end_time":"2024-02-10T05:23:03.973644","exception":false,"start_time":"2024-02-10T05:23:03.267517","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.032096,"end_time":"2024-02-10T05:23:04.038858","exception":false,"start_time":"2024-02-10T05:23:04.006762","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\nsubm_df[\"score\"] = y_pred","metadata":{"papermill":{"duration":0.053299,"end_time":"2024-02-10T05:23:04.124231","exception":false,"start_time":"2024-02-10T05:23:04.070932","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", subm_df[\"score\"].isnull().any())\nsubm_df.head()","metadata":{"papermill":{"duration":0.049359,"end_time":"2024-02-10T05:23:04.206170","exception":false,"start_time":"2024-02-10T05:23:04.156811","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.043978,"end_time":"2024-02-10T05:23:04.283155","exception":false,"start_time":"2024-02-10T05:23:04.239177","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}