{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":162470947,"sourceType":"kernelVersion"},{"sourceId":169866666,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":1479.820182,"end_time":"2024-02-10T05:23:05.644646","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-10T04:58:25.824464","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.006713,"end_time":"2024-02-10T04:58:28.609783","exception":false,"start_time":"2024-02-10T04:58:28.60307","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --no-index -Uq --find-links=/kaggle/input/lightautoml-038-dependencies lightautoml==0.3.8","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-03T12:02:45.352610Z","iopub.execute_input":"2024-04-03T12:02:45.353443Z","iopub.status.idle":"2024-04-03T12:05:06.974488Z","shell.execute_reply.started":"2024-04-03T12:02:45.353408Z","shell.execute_reply":"2024-04-03T12:05:06.972958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport lightgbm as lgb\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.303403,"end_time":"2024-02-10T04:58:34.920027","exception":false,"start_time":"2024-02-10T04:58:28.616624","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-03T12:05:09.351052Z","iopub.execute_input":"2024-04-03T12:05:09.351657Z","iopub.status.idle":"2024-04-03T12:05:16.966288Z","shell.execute_reply.started":"2024-04-03T12:05:09.351621Z","shell.execute_reply":"2024-04-03T12:05:16.965520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightautoml.automl.presets.tabular_presets import TabularAutoML\nfrom lightautoml.tasks import Task\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:06:44.271111Z","iopub.execute_input":"2024-04-03T12:06:44.271889Z","iopub.status.idle":"2024-04-03T12:07:11.678761Z","shell.execute_reply.started":"2024-04-03T12:06:44.271856Z","shell.execute_reply":"2024-04-03T12:07:11.677608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data collection","metadata":{"papermill":{"duration":0.006776,"end_time":"2024-02-10T04:58:34.934015","exception":false,"start_time":"2024-02-10T04:58:34.927239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:08:06.827652Z","iopub.execute_input":"2024-04-03T12:08:06.828527Z","iopub.status.idle":"2024-04-03T12:08:06.844087Z","shell.execute_reply.started":"2024-04-03T12:08:06.828496Z","shell.execute_reply":"2024-04-03T12:08:06.843139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import utility script notebook with data collection functions. See notebook for details: [Home Credit: baseline - DATA](https://www.kaggle.com/code/andreynesterov/home-credit-baseline-data)","metadata":{"papermill":{"duration":0.006661,"end_time":"2024-02-10T04:58:34.947624","exception":false,"start_time":"2024-02-10T04:58:34.940963","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import home_credit_baseline_data1 as data_nb","metadata":{"papermill":{"duration":0.025934,"end_time":"2024-02-10T04:58:34.98032","exception":false,"start_time":"2024-02-10T04:58:34.954386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-03T12:08:12.228942Z","iopub.execute_input":"2024-04-03T12:08:12.229599Z","iopub.status.idle":"2024-04-03T12:08:12.238838Z","shell.execute_reply.started":"2024-04-03T12:08:12.229569Z","shell.execute_reply":"2024-04-03T12:08:12.237906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_files = [\n    \"_static_cb_0.parquet\",\n    \"_static_0_*.parquet\",\n    \"_applprev_1_*.parquet\",\n    \"_tax_registry_a_1.parquet\",\n    \"_tax_registry_b_1.parquet\",\n    \"_tax_registry_c_1.parquet\",\n    \"_other_1.parquet\",\n    \"_person_1.parquet\",\n    \"_deposit_1.parquet\",\n    \"_debitcard_1.parquet\",\n    \"_credit_bureau_b_1.parquet\",\n    \"_credit_bureau_b_2.parquet\",\n]\nbase_agg = data_nb.Aggregator(\n#     num_aggregators = [pl.max, pl.min, pl.first, pl.last, pl.mean],\n#     str_aggregators = [pl.max, pl.min, pl.first, pl.last],\n#     group_aggregators = [pl.max, pl.min, pl.first, pl.last],\n    num_aggregators=[pl.max, pl.min,pl.mean],\n    date_aggregators=[pl.max,pl.min],\n    str_aggregators=[pl.max,pl.n_unique],\n    other_aggregators=[pl.max,pl.min],\n    group_aggregators=[pl.max],\n    str_mode = True\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:09:31.961782Z","iopub.execute_input":"2024-04-03T12:09:31.962717Z","iopub.status.idle":"2024-04-03T12:09:31.971187Z","shell.execute_reply.started":"2024-04-03T12:09:31.962671Z","shell.execute_reply":"2024-04-03T12:09:31.970436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = data_nb.prepare_df(base_files,data_nb.CFG.train_dir,base_agg)\ntrain_df = pd.read_parquet('/kaggle/usr/lib/home_credit_baseline_data/train_base.parquet')\ncat_cols = list(train_df.select_dtypes(\"category\").columns)\ndisplay(train_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":142.824103,"end_time":"2024-02-10T05:00:57.811638","exception":false,"start_time":"2024-02-10T04:58:34.987535","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-03T12:08:16.791108Z","iopub.execute_input":"2024-04-03T12:08:16.792054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(cat_cols)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.027724,"end_time":"2024-02-10T05:00:57.856133","exception":false,"start_time":"2024-02-10T05:00:57.828409","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-04-03T12:08:29.128600Z","iopub.execute_input":"2024-04-03T12:08:29.129327Z","iopub.status.idle":"2024-04-03T12:08:29.136693Z","shell.execute_reply.started":"2024-04-03T12:08:29.129293Z","shell.execute_reply":"2024-04-03T12:08:29.135665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(base_files,data_nb.CFG.test_dir,base_agg,cat_cols=cat_cols, mode=\"test\", train_cols=train_df.columns)\ndisplay(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.993087,"end_time":"2024-02-10T05:00:58.866216","exception":false,"start_time":"2024-02-10T05:00:57.873129","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-03T12:09:34.957298Z","iopub.execute_input":"2024-04-03T12:09:34.957688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop columns","metadata":{"papermill":{"duration":0.025427,"end_time":"2024-02-10T05:00:58.917438","exception":false,"start_time":"2024-02-10T05:00:58.892011","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_cols = []\ndrop_cols_startwith = [\"mean_\", \"first_\"]\nfor name_prefix in drop_cols_startwith:\n    cols_names = train_df.columns[train_df.columns.str.startswith(name_prefix)]\n    drop_cols += cols_names.to_list()\ndisplay(drop_cols)\ndisplay(len(drop_cols))","metadata":{"papermill":{"duration":0.032795,"end_time":"2024-02-10T05:00:58.9758","exception":false,"start_time":"2024-02-10T05:00:58.943005","status":"completed"},"scrolled":true,"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-03T12:09:48.653508Z","iopub.execute_input":"2024-04-03T12:09:48.653885Z","iopub.status.idle":"2024-04-03T12:09:48.663983Z","shell.execute_reply.started":"2024-04-03T12:09:48.653856Z","shell.execute_reply":"2024-04-03T12:09:48.663040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(columns=drop_cols, inplace=True)\ntest_df.drop(columns=drop_cols, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:09:51.133815Z","iopub.execute_input":"2024-04-03T12:09:51.134165Z","iopub.status.idle":"2024-04-03T12:09:52.503490Z","shell.execute_reply.started":"2024-04-03T12:09:51.134139Z","shell.execute_reply":"2024-04-03T12:09:52.502408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:09:53.609974Z","iopub.execute_input":"2024-04-03T12:09:53.610326Z","iopub.status.idle":"2024-04-03T12:09:53.616771Z","shell.execute_reply.started":"2024-04-03T12:09:53.610300Z","shell.execute_reply":"2024-04-03T12:09:53.615720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = data_nb.reduce_mem_usage(train_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:09:55.640013Z","iopub.execute_input":"2024-04-03T12:09:55.640925Z","iopub.status.idle":"2024-04-03T12:11:14.269587Z","shell.execute_reply.started":"2024-04-03T12:09:55.640883Z","shell.execute_reply":"2024-04-03T12:11:14.268643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.reduce_mem_usage(test_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:13:53.314950Z","iopub.execute_input":"2024-04-03T12:13:53.315689Z","iopub.status.idle":"2024-04-03T12:13:53.509692Z","shell.execute_reply.started":"2024-04-03T12:13:53.315660Z","shell.execute_reply":"2024-04-03T12:13:53.508804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:14:01.267988Z","iopub.execute_input":"2024-04-03T12:14:01.268584Z","iopub.status.idle":"2024-04-03T12:14:01.552696Z","shell.execute_reply.started":"2024-04-03T12:14:01.268553Z","shell.execute_reply":"2024-04-03T12:14:01.551781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.025165,"end_time":"2024-02-10T05:00:59.026486","exception":false,"start_time":"2024-02-10T05:00:59.001321","status":"completed"},"tags":[]}},{"cell_type":"code","source":"N_THREADS = 4\nN_FOLDS = 5\nRANDOM_STATE = 2024\nTIMEOUT = 10000\nADVANCED_ROLES = False\nUSE_QNT = False\nUSE_PLR = False\nTRAIN_BS = 128\nEPOCHS = 5\nTARGET_NAME = 'target'\n\nnp.random.seed(RANDOM_STATE)\ntorch.set_num_threads(N_THREADS)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:14:03.025479Z","iopub.execute_input":"2024-04-03T12:14:03.025845Z","iopub.status.idle":"2024-04-03T12:14:03.035478Z","shell.execute_reply.started":"2024-04-03T12:14:03.025817Z","shell.execute_reply":"2024-04-03T12:14:03.034723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_class(x, task, reader):\n    if task.name == 'multiclass':\n        return reader[x]\n    else:\n        return x\n\nmapped = np.vectorize(map_class)\n\ndef score(task, y_true, y_pred):\n    if task.name == 'binary':\n        return roc_auc_score(y_true, y_pred)\n    elif task.name == 'multiclass':\n        return log_loss(y_true, y_pred)\n    elif task.name == 'reg' or task.name == 'multi:reg':\n        return mean_absolute_error(y_true, y_pred)\n    else:\n        raise 'Task is not correct.'\n        \ndef take_pred_from_task(pred, task):\n    if task.name == 'binary' or task.name == 'reg':\n        return pred[:, 0]\n    elif task.name == 'multiclass' or task.name == 'multi:reg':\n        return pred\n    else:\n        raise 'Task is not correct.'\n        \ndef use_plr(USE_PLR):\n    if USE_PLR:\n        return \"plr\"\n    else:\n        return \"cont\"","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:14:06.497472Z","iopub.execute_input":"2024-04-03T12:14:06.497898Z","iopub.status.idle":"2024-04-03T12:14:06.506610Z","shell.execute_reply.started":"2024-04-03T12:14:06.497870Z","shell.execute_reply":"2024-04-03T12:14:06.505538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task = Task(\n    'binary', \n    loss = 'logloss', \n    metric = 'auc'\n)\nroles = {\n    'target': TARGET_NAME,\n    'group': \"WEEK_NUM\",\n    'drop': ['case_id', \"WEEK_NUM\"],\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-03T12:14:08.732260Z","iopub.execute_input":"2024-04-03T12:14:08.732629Z","iopub.status.idle":"2024-04-03T12:14:08.744564Z","shell.execute_reply.started":"2024-04-03T12:14:08.732600Z","shell.execute_reply":"2024-04-03T12:14:08.743636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nalgo = 'denselight'\nautoml_model = TabularAutoML(\n    task = task, \n    timeout = TIMEOUT,\n    cpu_limit = N_THREADS,\n    gpu_ids='0',\n    general_params = {\"use_algos\": [[algo]]}, # ['nn', 'mlp', 'dense', 'denselight', 'resnet', 'snn', 'node', 'autoint', 'fttransformer'] or custom torch model\n    nn_params = {\n        \"n_epochs\": EPOCHS, \n        \"bs\": TRAIN_BS, \n        \"num_workers\": 0, \n        \"path_to_save\": None, \n        \"freeze_defaults\": True,\n        \"cont_embedder\": use_plr(USE_PLR),\n    },\n    nn_pipeline_params = {\n        \"use_qnt\": USE_QNT, \n        \"use_te\": False\n    },\n    reader_params = {\n        'n_jobs': N_THREADS, \n        'cv': N_FOLDS, \n        'random_state': RANDOM_STATE, \n        'advanced_roles': ADVANCED_ROLES\n    },\n)\n\noof_pred = automl_model.fit_predict(train_df, roles = roles, verbose = 3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-03T07:05:47.023092Z","iopub.execute_input":"2024-04-03T07:05:47.023662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_pred.data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = score(\n    task,\n    mapped(train_df[TARGET_NAME].values, task, automl_model.reader.class_mapping),\n    take_pred_from_task(oof_pred.data, task)\n)\nprint(\"CV roc_auc_oof: \", oof)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(oof_pred.data, 'denselight_oof_preds.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(automl_model, 'denselight_model.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump((train_df.columns, cat_cols, drop_cols), \"train_cat_columns.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oof_pred\ngc.collect()","metadata":{"papermill":{"duration":0.163109,"end_time":"2024-02-10T05:23:03.097082","exception":false,"start_time":"2024-02-10T05:23:02.933973","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"papermill":{"duration":0.031773,"end_time":"2024-02-10T05:23:03.16254","exception":false,"start_time":"2024-02-10T05:23:03.130767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=100000):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        batch_probs = model.predict(X_batch).data.squeeze()\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"papermill":{"duration":0.041537,"end_time":"2024-02-10T05:23:03.235674","exception":false,"start_time":"2024-02-10T05:23:03.194137","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred = pd.Series(predict_proba_in_batches(automl_model, X_test), index=X_test.index)\ny_pred[:10]","metadata":{"papermill":{"duration":0.706127,"end_time":"2024-02-10T05:23:03.973644","exception":false,"start_time":"2024-02-10T05:23:03.267517","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.032096,"end_time":"2024-02-10T05:23:04.038858","exception":false,"start_time":"2024-02-10T05:23:04.006762","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\nsubm_df[\"score\"] = y_pred","metadata":{"papermill":{"duration":0.053299,"end_time":"2024-02-10T05:23:04.124231","exception":false,"start_time":"2024-02-10T05:23:04.070932","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", subm_df[\"score\"].isnull().any())\nsubm_df.head()","metadata":{"papermill":{"duration":0.049359,"end_time":"2024-02-10T05:23:04.20617","exception":false,"start_time":"2024-02-10T05:23:04.156811","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.043978,"end_time":"2024-02-10T05:23:04.283155","exception":false,"start_time":"2024-02-10T05:23:04.239177","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}