{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7474185,"sourceType":"datasetVersion","datasetId":4351114},{"sourceId":7570020,"sourceType":"datasetVersion","datasetId":4021289},{"sourceId":7933160,"sourceType":"datasetVersion","datasetId":4663194},{"sourceId":8398362,"sourceType":"datasetVersion","datasetId":4442667},{"sourceId":8398371,"sourceType":"datasetVersion","datasetId":4662763},{"sourceId":8398367,"sourceType":"datasetVersion","datasetId":4438933},{"sourceId":33095,"sourceType":"modelInstanceVersion","modelInstanceId":27710},{"sourceId":33096,"sourceType":"modelInstanceVersion","modelInstanceId":27711}],"dockerImageVersionId":30683,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Training Notebooks\n- lgb https://www.kaggle.com/code/sunghoshim/home-credit-lgb-train-copy\n- cat https://www.kaggle.com/sunghoshim/home-credit-cat-train-copy\n\n## Reference\n- https://www.kaggle.com/code/xiaoleilian/home-credit-ensemble-infer-lgb-cat\n","metadata":{}},{"cell_type":"code","source":"import os\nos.system('pip install --force-reinstall /kaggle/input/homecredit-code/homecredit-0.1-py3-none-any.whl')\n\nfrom homecredit.setup_env import setup_environment\nsetup_environment()\nimport gc\n\nimport datetime as dt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.pipeline import Pipeline\nfrom tqdm.auto import tqdm\n\nfrom homecredit.models.tree import LGBM, CBM\nfrom homecredit.pipeline import FullPipeline, PipelineCV\nfrom homecredit.config import RANDOM_SEED, COL_TARGET, COL_ID, COL_DATE, COL_WEEK, PATH_DATA\nfrom homecredit.utils import weighted_rank_average","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:55:18.717120Z","iopub.execute_input":"2024-05-13T14:55:18.717526Z","iopub.status.idle":"2024-05-13T14:56:10.553519Z","shell.execute_reply.started":"2024-05-13T14:55:18.717490Z","shell.execute_reply":"2024-05-13T14:56:10.551507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_model(model_name, df, batch_size=10**6):\n    pipeline = FullPipeline(\n        None,\n        run_name=\"full\",\n        name=model_name,\n        load_model=True,\n        features=None\n    )\n    pipeline.fit(verbose=True)\n#     cv = PipelineCV(pipeline, n_splits=5)\n#     cv.load()\n    preds = pipeline.predict_proba_in_batches(df, batch_size=10**6)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:10.556494Z","iopub.execute_input":"2024-05-13T14:56:10.557247Z","iopub.status.idle":"2024-05-13T14:56:10.563188Z","shell.execute_reply.started":"2024-05-13T14:56:10.557210Z","shell.execute_reply":"2024-05-13T14:56:10.561923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Version baseline\nfrom homecredit.data.data_processor_2 import DataProcessor2\ndata_processor = DataProcessor2(\"version_1\")\ndf = data_processor.get_data(test=True, save_to_disc=False, load=False)\npreds_df = pd.DataFrame(index=df[\"case_id\"])\n\nmodel_names = [\n    'blend_lgb_run_13', 'blend_lgb_run_stable_400', 'blend_cb_run_407',\n    'blend_lgb_run_160', 'blend_lgb_run_109', #'lgb_run_version_14',\n]\nfor model_name in tqdm(model_names):\n    preds_df[model_name] = predict_model(model_name, df, batch_size=10**6)\n\n## SUBMISSION FILE\nsubm_df = pd.read_csv(\n    os.path.join(PATH_DATA, \"sample_submission.csv\"),\n    dtype={COL_ID: int}\n).set_index(COL_ID)\n\ncol = \"refreshdate_3813885D_1_min_creditbureau1\"\ndf[\"date\"] = dt.datetime(2019,1,3) - pd.to_timedelta(df[col], unit='d')\nsubm_df = subm_df.merge(df[[\"case_id\", \"date\"]], how=\"left\", on=\"case_id\").set_index(\"case_id\")\n### \n\ndel df, data_processor\ngc.collect()\n\n### Version 500\nfrom homecredit.data.data_processor_3 import DataProcessor3\ndata_processor = DataProcessor3(\"version_1\")\ndf = data_processor.get_data(test=True, save_to_disc=False, load=False)\n\nmodel_names = [\n    \"lgb_version_500_0\",\n]\nfor model_name in tqdm(model_names):\n    preds_df[model_name] = predict_model(model_name, df, batch_size=10**6)\n\ndel df, data_processor\ngc.collect()\n\n### Version 58\nfrom homecredit.data.data_processor_1 import DataProcessor1\ndata_processor = DataProcessor1(\"version_1\")\ndf = data_processor.get_data(test=True, save_to_disc=False, load=False)\n\nmodel_names = [\n    'blend_cb_run_12', 'blend_lgb_run_12', \n]\n        \nfor model_name in model_names:\n    preds_df[model_name] = predict_model(model_name, df, batch_size=10**6)\n    \ndel df, data_processor\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:10.564776Z","iopub.execute_input":"2024-05-13T14:56:10.565107Z","iopub.status.idle":"2024-05-13T14:56:18.554732Z","shell.execute_reply.started":"2024-05-13T14:56:10.565081Z","shell.execute_reply":"2024-05-13T14:56:18.553486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nfrom pathlib import Path\nimport gc\nfrom glob import glob\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nROOT = '/kaggle/input/home-credit-credit-risk-model-stability'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-13T14:56:18.557054Z","iopub.execute_input":"2024-05-13T14:56:18.557405Z","iopub.status.idle":"2024-05-13T14:56:18.566280Z","shell.execute_reply.started":"2024-05-13T14:56:18.557380Z","shell.execute_reply":"2024-05-13T14:56:18.564826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n                if isnull > 0.7:\n                    df = df.drop(col)\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df\n\n\n\nclass Aggregator:\n    # Please add or subtract features yourself, be aware that too many features will take up too much space.\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        expr_var = [pl.var(col).alias(f\"var_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean \n\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean \n\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        return expr_max + expr_last  # +expr_count\n\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return expr_max + expr_last\n\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return expr_max + expr_last\n\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:18.569696Z","iopub.execute_input":"2024-05-13T14:56:18.570018Z","iopub.status.idle":"2024-05-13T14:56:18.597596Z","shell.execute_reply.started":"2024-05-13T14:56:18.569994Z","shell.execute_reply":"2024-05-13T14:56:18.596033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols\n\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:18.599598Z","iopub.execute_input":"2024-05-13T14:56:18.600228Z","iopub.status.idle":"2024-05-13T14:56:18.620741Z","shell.execute_reply.started":"2024-05-13T14:56:18.600160Z","shell.execute_reply":"2024-05-13T14:56:18.619322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Models","metadata":{}},{"cell_type":"code","source":"lgb_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/notebook_info.joblib')\nprint(f\"- [lgb] notebook_start_time: {lgb_notebook_info['notebook_start_time']}\")\nprint(f\"- [lgb] description: {lgb_notebook_info['description']}\")\n\ncols = lgb_notebook_info['cols']\ncat_cols = lgb_notebook_info['cat_cols']\nprint(f\"- [lgb] len(cols): {len(cols)}\")\nprint(f\"- [lgb] len(cat_cols): {len(cat_cols)}\")\n\nlgb_models = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/lgb_models.joblib')\nlgb_models","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:18.622304Z","iopub.execute_input":"2024-05-13T14:56:18.622676Z","iopub.status.idle":"2024-05-13T14:56:18.960756Z","shell.execute_reply.started":"2024-05-13T14:56:18.622648Z","shell.execute_reply":"2024-05-13T14:56:18.959319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/notebook_info.joblib')\nprint(f\"- [cat] notebook_start_time: {cat_notebook_info['notebook_start_time']}\")\nprint(f\"- [cat] description: {cat_notebook_info['description']}\")\n\ncat_models = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/cat_models.joblib')\ncat_models","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:18.962382Z","iopub.execute_input":"2024-05-13T14:56:18.962741Z","iopub.status.idle":"2024-05-13T14:56:20.753104Z","shell.execute_reply.started":"2024-05-13T14:56:18.962713Z","shell.execute_reply":"2024-05-13T14:56:20.751462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare df_test","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"\n\ndata_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TEST_DIR / \"test_applprev_2.parquet\", 2),\n        read_file(TEST_DIR / \"test_person_2.parquet\", 2)\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:20.756743Z","iopub.execute_input":"2024-05-13T14:56:20.757131Z","iopub.status.idle":"2024-05-13T14:56:20.966078Z","shell.execute_reply.started":"2024-05-13T14:56:20.757105Z","shell.execute_reply":"2024-05-13T14:56:20.964636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)\ndel data_store\ngc.collect()\n\ndf_test = df_test.select(['case_id'] + cols)\n\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test = reduce_mem_usage(df_test)\ndf_test = df_test.set_index('case_id')\nprint(\"test data shape:\\t\", df_test.shape)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:20.969262Z","iopub.execute_input":"2024-05-13T14:56:20.969664Z","iopub.status.idle":"2024-05-13T14:56:21.511729Z","shell.execute_reply.started":"2024-05-13T14:56:20.969642Z","shell.execute_reply":"2024-05-13T14:56:21.509053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Voting Model","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n     \n    def predict_proba(self, X): \n        preds_df_v = pd.DataFrame(index=np.arange(len(X)), columns=range(10))\n        i = 0\n        # lgb\n        for estimator in self.estimators[:5]:\n            preds_df_v[i] = estimator.predict_proba(X)[:,1]\n            i+=1\n        # cat        \n        X[cat_cols] = X[cat_cols].astype(str)\n        for estimator in self.estimators[-5:]:\n            preds_df_v[i] = estimator.predict_proba(X)[:,1]\n            i+=1\n        weights = np.ones(10)/10\n#         weights[-5:] = 2\n        print(weights)\n        preds = (preds_df_v * weights).sum(axis=1)\n        return preds","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:21.513168Z","iopub.execute_input":"2024-05-13T14:56:21.513470Z","iopub.status.idle":"2024-05-13T14:56:21.524363Z","shell.execute_reply.started":"2024-05-13T14:56:21.513450Z","shell.execute_reply":"2024-05-13T14:56:21.522501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VotingModel(lgb_models + cat_models)\ny_pred = pd.Series(model.predict_proba(df_test).values, index=df_test.index)\npreds_df[\"blend_kaggle_ens\"] = y_pred\nlen(model.estimators)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:21.526040Z","iopub.execute_input":"2024-05-13T14:56:21.526441Z","iopub.status.idle":"2024-05-13T14:56:21.901699Z","shell.execute_reply.started":"2024-05-13T14:56:21.526411Z","shell.execute_reply":"2024-05-13T14:56:21.900274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = np.ones(preds_df.shape[1])\nweights[-1] = 4\nweights = weights / sum(weights)\nprint(pd.DataFrame(weights, index=preds_df.columns))\n# preds = weighted_rank_average(preds_df, weights)\npreds = (preds_df * weights).sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:21.903890Z","iopub.execute_input":"2024-05-13T14:56:21.904277Z","iopub.status.idle":"2024-05-13T14:56:21.914672Z","shell.execute_reply.started":"2024-05-13T14:56:21.904246Z","shell.execute_reply":"2024-05-13T14:56:21.913239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_min = dt.datetime(2020, 10, 6)\ndate_max = subm_df[\"date\"].max()\ndate_mid = dt.datetime(2021, 5, 31)\nprint(f\"{date_min:%d-%m-%Y} : {date_mid:%d-%m-%Y} : {date_max:%d-%m-%Y}\")\n\nINVERSE = False\nPENALTY = True\nCOND = False\n# if date_max < dt.datetime(2023, 1, 20):\n#     INVERSE = True\n#     PENALTY = True\n#     COND = True\n\nprint(f\"Inverse {INVERSE}, penalty {PENALTY}, cond {COND}\")\n    \nsubm_df[\"score\"] = preds\nsubm_df[\"cond\"] = (subm_df[\"date\"] < date_mid) & (subm_df[\"date\"] >= date_min)\nprint(\"-\"*40)\nprint(subm_df)\n    \nif PENALTY:\n    subm_df.loc[subm_df[\"cond\"], \"score\"] = np.maximum(subm_df.loc[subm_df[\"cond\"], \"score\"]-0.038, 0)\n    \nprint(\"-\"*40)\nprint(subm_df)\n    \nif INVERSE:\n    subm_df[\"score\"] = 1-subm_df[\"score\"]\n    \nprint(\"-\"*40)\nprint(subm_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:21.916516Z","iopub.execute_input":"2024-05-13T14:56:21.917056Z","iopub.status.idle":"2024-05-13T14:56:21.938741Z","shell.execute_reply.started":"2024-05-13T14:56:21.917028Z","shell.execute_reply":"2024-05-13T14:56:21.937321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.drop(columns=[\"date\", \"cond\"], inplace=True)\n\nsubm_df.to_csv(\"submission.csv\")\nprint(subm_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:56:21.941170Z","iopub.execute_input":"2024-05-13T14:56:21.941880Z","iopub.status.idle":"2024-05-13T14:56:21.962746Z","shell.execute_reply.started":"2024-05-13T14:56:21.941844Z","shell.execute_reply":"2024-05-13T14:56:21.960735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}