{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. IMPORTS","metadata":{}},{"cell_type":"markdown","source":"**1.1 IMPORTS ONE**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nimport sys\n\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport gc\nfrom glob import glob\nimport os\n\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:14.317280Z","iopub.execute_input":"2024-04-29T06:13:14.317888Z","iopub.status.idle":"2024-04-29T06:13:14.732526Z","shell.execute_reply.started":"2024-04-29T06:13:14.317858Z","shell.execute_reply":"2024-04-29T06:13:14.731745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1.2 IMPORTS TWO**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nfrom imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\nfrom sklearn.model_selection import train_test_split\nfrom catboost import CatBoostClassifier, Pool\n\n# smote = SMOTE()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:14.734084Z","iopub.execute_input":"2024-04-29T06:13:14.734467Z","iopub.status.idle":"2024-04-29T06:13:17.187056Z","shell.execute_reply.started":"2024-04-29T06:13:14.734441Z","shell.execute_reply":"2024-04-29T06:13:17.186100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. CLASSES","metadata":{}},{"cell_type":"markdown","source":"**2.1 CLASS - AGGREGATOR**","metadata":{}},{"cell_type":"code","source":"\nclass Aggregator:\n    #Please add or subtract features yourself, be aware that too many features will take up too much space.\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        \n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).cast(pl.Int64).alias(f\"mean_{col}\") for col in cols]\n\n\n        # expr_myMetric = [((pl.median(col) + pl.mean(col) + 2*pl.max(col))/4).cast(pl.Int64).alias(f\"myMetric_{col}\") for col in cols]\n\n        return expr_max +expr_last+expr_mean\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n\n        return  expr_max +expr_last+expr_mean\n    \n    #take max for string data types\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        #expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        return expr_max +expr_last\n    \n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs\n    \n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.188338Z","iopub.execute_input":"2024-04-29T06:13:17.188862Z","iopub.status.idle":"2024-04-29T06:13:17.205473Z","shell.execute_reply.started":"2024-04-29T06:13:17.188833Z","shell.execute_reply":"2024-04-29T06:13:17.204437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.2 CLASS - PIPELINE**","metadata":{}},{"cell_type":"code","source":"# NOTE - The Pipeline functions here are used on the polars dataframe\nclass Pipeline:\n\n    #PENDING -- Handle cases for T,L\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df): #PENDING, CHECK ALL THE DATES\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\")) #!!?\n                df = df.with_columns(pl.col(col).dt.total_days().cast(pl.Int64)) # t - t-1 #modified\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    #pass a list as argument and dont drop the columns if present in the list\n    def filter_cols(df, cols_must_present=[]):\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) and (col not in cols_must_present):\n                isnull = df[col].is_null().mean()\n                if (isnull!= None) and (isnull > 0.7):\n                    df = df.drop(col)\n\n        #uncomment later\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df\n    \n\n\n    #Group together columns based on the correlation, and select the top column from each group\n    def filter_correlated_columns(df):\n        #reduce the number of groups\n        def reduce_group(grps):\n            use = []\n            for g in grps:\n                mx = 0; vx = g[0]\n                for gg in g:\n                    n = df[gg].n_unique()\n                    if n>mx:\n                        mx = n\n                        vx = gg\n                    #print(str(gg)+'-'+str(n),', ',end='')\n                use.append(vx)\n                #print()\n            # print('Use these',use)\n            return use\n        \n        def group_columns_by_correlation(matrix, threshold=0.7): #modified\n            matrix=matrix.drop_nulls()\n            # print('matrix',matrix)\n            correlation_matrix = matrix.corr()\n\n            # print('correlation_matrix',correlation_matrix)\n\n            # dict to store 'columnname' as key and index as 'value'\n            index_dict = {}\n\n            # Iterate through the list and fill the dictionary\n            for index, element in enumerate(matrix.columns):\n                index_dict[element] = index\n\n            # print('index dict',index_dict)\n\n            groups = []\n            remaining_cols = list(matrix.columns)\n            # print('correlation matrix',correlation_matrix)\n\n            while remaining_cols:\n                col = remaining_cols.pop(0)\n                group = [col]\n                correlated_cols = [col]\n                \n                for c in remaining_cols:\n                    row = index_dict[c]\n                    corr_value = correlation_matrix.select(pl.col(col))\n                    corr_value = corr_value[col][row]\n                    # print('corr_value',corr_value)\n                    if corr_value >= threshold:\n                        group.append(c)\n                        correlated_cols.append(c)\n                groups.append(group)\n                remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n            \n            return groups\n\n        #exclude string/object datatpyes - add more to the list like object Boolean\n        df_changed = df.select(pl.exclude([pl.String,pl.Date,pl.Boolean]))\n\n        #create a masked polar dataframe for df_changed\n        df_nans = {}\n        for col in df_changed.columns:\n            df_nans[col] = df_changed[col].is_null()\n        df_nans = pl.DataFrame(df_nans)\n        # print(df_nans)\n\n        #dict - key= number of missing values,, value=list having column names\n        nans_groups={}\n        for col in df_nans.columns:\n            cur_group = df_nans[col].sum()\n            try:\n                nans_groups[cur_group].append(col)\n            except:\n                nans_groups[cur_group]=[col]\n\n        #uses is a list which have the selected columns for the model\n        uses = []\n        for k,v in nans_groups.items():\n            if len(v)>1:\n                    # print('level 1')\n                    Vs = nans_groups[k]\n                    grps= group_columns_by_correlation(df[Vs], threshold=0.8)\n\n                    # print('grps',grps)\n                    use=reduce_group(grps)\n                    uses=uses+use\n            else:\n                uses=uses+v\n            print('####### NAN count =',k)\n        \n\n        #add the columns which were excluded earlier, while computing the correlation\n        # Define the data types you want to select\n        data_types_to_select = ['String', 'Date','Boolean']  # Adjust data types as needed like Boolean, matchin the upper list\n        # Get column names of selected data types\n        excluded_cats = [col for col in df.columns if str(df[col].dtype) in data_types_to_select]\n        uses=uses+excluded_cats\n\n        print('columns that are not dropped after correlation are')\n        print(uses)\n\n        return df.select(uses), excluded_cats\n\n\n    def non_numeric_to_string(df):\n        non_numeric_types = ['Boolean',]\n        for col in df.columns:\n            if str(df[col].dtype) in non_numeric_types:\n                df = df.with_columns(pl.col(col).cast(pl.String))\n\n        return df\n\n\n\n\n    def drop_rows_with_nulls(df):\n        df = df.drop_nulls()\n        return df\n    \n    def normalize_cols(df):\n   \n        def normalize_column(column):\n            min_val = column.min()\n            max_val = column.max()\n            return (column - min_val) / (max_val - min_val)\n        \n        # Apply Min-Max scaling to each numerical column (both Float64 and Int64)\n        normalized_df = df.select([\n            normalize_column(pl.col(col)).alias(col) \n            if df[col].dtype in [pl.Float64, pl.Int64] and col not in ['MONTH', 'num_group2', 'num_group1', 'date_decision', 'WEEK_NUM', 'target', 'case_id']\n            else pl.col(col)\n            for col in df.columns\n        ])\n        \n        return normalized_df\n    \n    #Instead of encode columns use, cat_column and then pass them as categories using pandas\n    # X_train[cat_cols].astype(\"category\") --- used for pandas\n    def encode_cols(df):\n        for col in df.columns:\n            dict1={}\n            freq = df[col].n_unique()\n            \n            \n            categories = sorted(df[col].unique().to_list())\n            \n            if(len(categories)==1):\n                continue\n\n            if freq < 200 and df[col].dtype == pl.String:\n                for i in range(freq):\n                    dict1[categories[i]]=i\n\n                new_col_name = col\n\n\n                df = df.with_columns( \n                        df[col].map_elements(lambda x: dict1.get(x), return_dtype=pl.UInt8).alias(new_col_name)\n                )\n\n            # print(dict1)\n        return df\n\n    #this function should fill the missing values with a valid metric\n    def handle_missing_values(df):\n        pass\n    \n                \n\n            \n\n\n    \n# print(sys.getsizeof(Pipeline))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.207632Z","iopub.execute_input":"2024-04-29T06:13:17.207924Z","iopub.status.idle":"2024-04-29T06:13:17.242122Z","shell.execute_reply.started":"2024-04-29T06:13:17.207888Z","shell.execute_reply":"2024-04-29T06:13:17.241220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.3 CLASS - VOTING**","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        # mode_result = mode(y_preds).mode\n        return np.mean(y_preds, axis=0)\n     \n    def predict_proba(self, X):      #Edit this in future to improve performance/score #modified\n        # lgb\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]\n        \n        #cat        \n        X[cat_cols] = X[cat_cols].astype(str)\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:]]\n        \n        return np.mean(y_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.243104Z","iopub.execute_input":"2024-04-29T06:13:17.243404Z","iopub.status.idle":"2024-04-29T06:13:17.254300Z","shell.execute_reply.started":"2024-04-29T06:13:17.243369Z","shell.execute_reply":"2024-04-29T06:13:17.253494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# 3. COMMON FUNCTIONS","metadata":{}},{"cell_type":"markdown","source":"**3.1 FUNCTION - READ FILES**","metadata":{}},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.255240Z","iopub.execute_input":"2024-04-29T06:13:17.255493Z","iopub.status.idle":"2024-04-29T06:13:17.266305Z","shell.execute_reply.started":"2024-04-29T06:13:17.255472Z","shell.execute_reply":"2024-04-29T06:13:17.265507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.2 FUNCTION - REDUCE MEMORY OF POLARS DATAFRAME**","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage_polars(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.estimated_size() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = str(df[col].dtype)\n        # print(col,'coltype before',col_type)\n        if  col_type in  ['Int64', 'Float64']:\n            #perform below operations\n            c_min = df[col].min()\n            c_max = df[col].max()\n\n            if (c_max is None) or (c_min is None):\n                print(\"skipping the column\",col)\n                continue\n\n            if col_type == 'Int64' or col_type == 'Float64':\n                # print('reduce mem if condition')\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int8))\n                    # print('reduce mem if condition - Int8')\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int16))\n                    # print('reduce mem if condition Int16')\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int32))\n                    # print('reduce mem if condition Int32')\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int64))\n                    # print('reduce mem if condition Int64')\n\n            # elif col_type == 'Float64':\n            #     if c_max < np.finfo(np.float32).max:\n            #         df = df.with_columns(pl.col(col).cast(pl.Float32))\n            #     else:\n            #         df = df.with_columns(pl.col(col).cast(pl.Float64))\n        # print(col,'coltype after',df[col].dtype)\n        # print('\\n')\n\n    end_mem = df.estimated_size() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    print(\"\\n\")\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.267402Z","iopub.execute_input":"2024-04-29T06:13:17.267720Z","iopub.status.idle":"2024-04-29T06:13:17.280475Z","shell.execute_reply.started":"2024-04-29T06:13:17.267690Z","shell.execute_reply":"2024-04-29T06:13:17.279348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.3 FUNCTION - UPDATE PARQUET FILES**","metadata":{}},{"cell_type":"code","source":"\ndef updateParquetFiles(data_store,cols_must_present=[]):\n    for value in data_store.values():\n        for item in value:\n            print(\"updateParquetFiles: reading parquet file from\",item[1]) #-- corresponds to the path of the parquet_file\n\n            if len(item)==2:\n                df = item[0](item[1]) #readfile/ readfiles happens here\n            elif len(item)==3:\n                df = item[0](item[1],item[2]) #readfile/ readfiles happens here\n            \n            #skip the empty file\n            # if df.shape[0] == 0:\n            #     continue\n\n            #filetering the colunmns of dataframe\n            # 1. you can perform filter_cols here\n            # 2. you can perform filter_cols in feature engineering\n            # df = df.pipe(Pipeline.filter_cols,cols_must_present)\n\n            #dropping the rows with missing values -- store this result in {folder_name}_inter1\n            # df = df.pipe(Pipeline.drop_rows_with_nulls)\n            #INSTEAD of dropping rows having the null values \n                # 1. retain the missing values/and handle it later - This method is selected\n                # 2. impute the missing values -- it is better to fill missing values after joining all tables/files\n\n            #reduce the dataframe memory size\n            # df = df.pipe(reduce_mem_usage_polars)\n\n            #WRITE df as parquet file to ../train_inter/ directory.\n            # 1. Get the filename from the path\n            filename = os.path.basename(item[1])\n            filename = filename.replace(\"_*\",\"\")\n            # Get the folder name from the path\n            folder_name = os.path.basename(os.path.dirname(item[1]))\n\n            # 2. Go one folder back\n            parent_directory = os.path.dirname(os.path.dirname(item[1]))\n\n            # 3. Create directory '{folder_name}_inter1' if not present\n            new_directory = os.path.join(parent_directory, f\"{folder_name}_inter1\")\n            \n            #replace 'input' to 'working'\n            new_directory = new_directory.replace('input','working')\n            \n            if not os.path.exists(new_directory):\n                os.makedirs(new_directory)\n\n            # 4. Create new path\n            new_path = os.path.join(new_directory, filename)\n            \n            print('updateParquetFiles: writing parquet file to',new_path)\n\n            # Write the DataFrame to a Parquet file in the new path\n            df.write_parquet(new_path)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.281729Z","iopub.execute_input":"2024-04-29T06:13:17.281994Z","iopub.status.idle":"2024-04-29T06:13:17.292693Z","shell.execute_reply.started":"2024-04-29T06:13:17.281973Z","shell.execute_reply":"2024-04-29T06:13:17.291780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.4 FUNCTION - REDUCE MEMORY OF PANDAS DATAFRAME**","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage_pandas(df_pandas):\n   \n    start_mem = df_pandas.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df_pandas.columns:\n        col_type = df_pandas[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df_pandas[col].min()\n            c_max = df_pandas[col].max()\n\n\n            if c_min is None or c_max is None:\n                print('hellooooooooooooooooooooooooooooooooooooooo')\n                continue\n\n\n            if str(col_type)[:5] == 'float':\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df_pandas[col] = df_pandas[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df_pandas[col] = df_pandas[col].astype(np.float32)\n                else:\n                    df_pandas[col] = df_pandas[col].astype(np.float64)\n\n\n            # if str(col_type)[:3] == 'int':\n            #     if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.int8)\n            #     elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.int16)\n            #     elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.int32)\n            #     elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.int64)  \n            # else:\n            #     if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.float16)\n            #     elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n            #         df_pandas[col] = df_pandas[col].astype(np.float32)\n            #     else:\n            #         df_pandas[col] = df_pandas[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df_pandas.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df_pandas","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.293706Z","iopub.execute_input":"2024-04-29T06:13:17.293981Z","iopub.status.idle":"2024-04-29T06:13:17.304993Z","shell.execute_reply.started":"2024-04-29T06:13:17.293959Z","shell.execute_reply":"2024-04-29T06:13:17.304159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. PATHS","metadata":{}},{"cell_type":"markdown","source":"**4.1 INITIALIZATION OF PATH*","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nROOT_INTER     = Path(\"/kaggle/working/home-credit-credit-risk-model-stability\")\n\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"\n\n#to store intermediate results, instead of keeping them in RAM\nTRAIN_INTER       = ROOT_INTER / \"parquet_files\" / \"train_inter\"\nTEST_INTER        = ROOT_INTER / \"parquet_files\" / \"test_inter\"\n\n#to store intermediate results, instead of keeping them in RAM\nTRAIN_INTER1       = ROOT_INTER / \"parquet_files\" / \"train_inter1\"\nTEST_INTER1        = ROOT_INTER / \"parquet_files\" / \"test_inter1\"","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.308731Z","iopub.execute_input":"2024-04-29T06:13:17.309229Z","iopub.status.idle":"2024-04-29T06:13:17.316508Z","shell.execute_reply.started":"2024-04-29T06:13:17.309196Z","shell.execute_reply":"2024-04-29T06:13:17.315648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(ROOT))\nprint('TRAIN_DIR',TRAIN_DIR)\nprint('TEST_DIR',TEST_DIR)\nprint('TRAIN_INTER',TRAIN_INTER)\nprint('TEST_INTER',TEST_INTER)\nprint('TRAIN_INTER1',TRAIN_INTER1)\nprint('TEST_INTER1',TEST_INTER1)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.317768Z","iopub.execute_input":"2024-04-29T06:13:17.318386Z","iopub.status.idle":"2024-04-29T06:13:17.325968Z","shell.execute_reply.started":"2024-04-29T06:13:17.318354Z","shell.execute_reply":"2024-04-29T06:13:17.325072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4.2 PREPROCESSING OF PATHS**","metadata":{}},{"cell_type":"code","source":"os.system(\"rm -rf \"+ str(ROOT_INTER))\nos.makedirs('/kaggle/working/home-credit-credit-risk-model-stability/parquet_files/',exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.327039Z","iopub.execute_input":"2024-04-29T06:13:17.327368Z","iopub.status.idle":"2024-04-29T06:13:17.497613Z","shell.execute_reply.started":"2024-04-29T06:13:17.327337Z","shell.execute_reply":"2024-04-29T06:13:17.496834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. TRAINING PART","metadata":{}},{"cell_type":"markdown","source":"**5.1 FUNCTION - FEATURE ENGINEERING FOR TRAIN DATA**","metadata":{}},{"cell_type":"code","source":"\ndef feature_eng_train(df_base, depth_0, depth_1, depth_2):\n\n    # print(df_base)\n    folderPath = os.path.dirname(os.path.dirname(df_base[0][1]))\n    folderName = os.path.basename(os.path.dirname(df_base[0][1]))\n\n    outputFolderName = folderName.replace(\"inter1\",\"inter\")\n    outputFolderPath = os.path.join(folderPath,outputFolderName)\n\n    print('outputFolderPath',outputFolderPath)\n\n    print('df_base[0][1]',df_base[0][1])\n    \n    #reading the parquet file using polars\n    df_base = pl.read_parquet(df_base[0][1])\n\n    #adding new columns to df_base\n    df_base = df_base.with_columns(\n        month_decision = pl.col(\"date_decision\").dt.month().cast(pl.Int8),\n        weekday_decision = pl.col(\"date_decision\").dt.weekday().cast(pl.Int8),\n    )\n    # print(df_base)\n\n    for i, list1 in enumerate(depth_0 + depth_1 + depth_2):\n        filename = str(list1[1])\n        \n        filename = filename.replace(\"_*\",\"\")\n        print('filename',filename)\n\n        if os.path.exists(filename):\n            df = pl.read_parquet(filename)\n            df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n\n    #handle dates     \n    df_base = df_base.pipe(Pipeline.handle_dates)\n\n    # #filter columns based on  the percentage of null values in each column - uncomment if needed\n    df_base = df_base.pipe(Pipeline.filter_cols)\n\n    # filter_correlated_columns\n    df_base, cat_cols = df_base.pipe(Pipeline.filter_correlated_columns)\n    \n        \n    outputFilename = os.path.join(outputFolderPath,'joined.parquet')\n    print('outputfilename',outputFilename)\n\n    #create director if not present\n    if not os.path.exists(outputFolderPath):\n        # Create the directory\n        os.makedirs(outputFolderPath)\n\n    #reduce memory of data frame\n    df_base = df_base.pipe(reduce_mem_usage_polars)\n\n    #convert non numerical data to string\n    df_base = df_base.pipe(Pipeline.non_numeric_to_string)\n\n    #writing the joined.parquet file to the outputFilename\n    df_base.write_parquet(outputFilename)\n\n    \n    return df_base, cat_cols\n\n    \n\n    # print('size of df_base',df_base)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.498862Z","iopub.execute_input":"2024-04-29T06:13:17.500186Z","iopub.status.idle":"2024-04-29T06:13:17.511751Z","shell.execute_reply.started":"2024-04-29T06:13:17.500151Z","shell.execute_reply":"2024-04-29T06:13:17.510827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5.2 FUNCTION - CREATE DATA STORE FOR TRAIN DATA**","metadata":{}},{"cell_type":"code","source":"def create_data_store_train(path):\n    d = {\n        \"df_base\": [\n            [read_file , path / \"train_base.parquet\"],\n        ],\n\n        \"depth_0\": [\n            [read_file , path / \"train_static_cb_0.parquet\"],\n            [read_files , path / \"train_static_0_*.parquet\"],\n        ],\n\n        \"depth_1\": [\n\n            [read_files , path / \"train_applprev_1_*.parquet\", 1],\n            [read_file , path / \"train_tax_registry_a_1.parquet\", 1],\n            [read_file , path / \"train_tax_registry_b_1.parquet\", 1],\n            [read_file , path / \"train_tax_registry_c_1.parquet\", 1],\n            [read_files , path / \"train_credit_bureau_a_1_*.parquet\", 1],\n            [read_file , path / \"train_credit_bureau_b_1.parquet\", 1],\n            [read_file , path / \"train_other_1.parquet\", 1],\n            [read_file , path / \"train_person_1.parquet\", 1],\n            [read_file , path / \"train_deposit_1.parquet\", 1],\n            [read_file , path / \"train_debitcard_1.parquet\", 1],\n        ],\n\n\n        \"depth_2\": [\n            [read_file , path / \"train_credit_bureau_b_2.parquet\", 2],\n            [read_files , path / \"train_credit_bureau_a_2_*.parquet\", 2],\n            [read_file , path / \"train_applprev_2.parquet\", 2],\n            [read_file , path / \"train_person_2.parquet\", 2],\n        ]\n\n\n    }\n\n\n    return d","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.512973Z","iopub.execute_input":"2024-04-29T06:13:17.513266Z","iopub.status.idle":"2024-04-29T06:13:17.523836Z","shell.execute_reply.started":"2024-04-29T06:13:17.513243Z","shell.execute_reply":"2024-04-29T06:13:17.522908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5.3 ACTIONS**","metadata":{}},{"cell_type":"code","source":"# create a train data store\ndata_store = create_data_store_train(TRAIN_DIR)\n\n# Update the intial parquet files\nupdateParquetFiles(data_store)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:13:17.524967Z","iopub.execute_input":"2024-04-29T06:13:17.525320Z","iopub.status.idle":"2024-04-29T06:16:09.755629Z","shell.execute_reply.started":"2024-04-29T06:13:17.525287Z","shell.execute_reply":"2024-04-29T06:16:09.754760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#apply feature engineering on train inter folder\ndf_base, cat_cols = feature_eng_train(**create_data_store_train(TRAIN_INTER1))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:16:09.756883Z","iopub.execute_input":"2024-04-29T06:16:09.757260Z","iopub.status.idle":"2024-04-29T06:17:11.176961Z","shell.execute_reply.started":"2024-04-29T06:16:09.757225Z","shell.execute_reply":"2024-04-29T06:17:11.176080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(df_base.dtypes))\nprint(len(cat_cols))\nprint(len(set(df_base.columns)))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:11.178016Z","iopub.execute_input":"2024-04-29T06:17:11.178283Z","iopub.status.idle":"2024-04-29T06:17:11.184157Z","shell.execute_reply.started":"2024-04-29T06:17:11.178259Z","shell.execute_reply":"2024-04-29T06:17:11.183202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_pandas = df_base.to_pandas()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:11.185540Z","iopub.execute_input":"2024-04-29T06:17:11.185883Z","iopub.status.idle":"2024-04-29T06:17:19.913618Z","shell.execute_reply.started":"2024-04-29T06:17:11.185853Z","shell.execute_reply":"2024-04-29T06:17:19.912814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(df_base_pandas.dtypes))\n\nstr_cols = []\nboolean_cols = []\n\nobj_cols = []\n\nbool_cols = []\n\nfor col in df_base.columns:\n    if str(df_base[col].dtype) in ['String']:\n        str_cols.append(col)\n\nfor col in df_base.columns:\n    if str(df_base[col].dtype) in ['Boolean']:\n        boolean_cols.append(col)\n\nfor col in df_base_pandas.columns:\n    if df_base_pandas[col].dtype in ['O']:\n        obj_cols.append(col)\n\nfor col in df_base_pandas.columns:\n    if df_base_pandas[col].dtype in ['bool']:\n        bool_cols.append(col)\n\n\nprint('len(str_cols)',len(str_cols))\nprint('bool_cols', bool_cols)\nprint('len(bool_cols)',len(bool_cols))\n\nprint('len(obj_cols)',len(obj_cols))\n\n\nprint(set(obj_cols)-set(str_cols))\n\n\nprint(len(boolean_cols))\n\nprint(set(str_cols)-set(cat_cols))\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:19.915139Z","iopub.execute_input":"2024-04-29T06:17:19.915529Z","iopub.status.idle":"2024-04-29T06:17:19.946568Z","shell.execute_reply.started":"2024-04-29T06:17:19.915496Z","shell.execute_reply":"2024-04-29T06:17:19.945561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_base\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:19.947851Z","iopub.execute_input":"2024-04-29T06:17:19.948428Z","iopub.status.idle":"2024-04-29T06:17:20.318171Z","shell.execute_reply.started":"2024-04-29T06:17:19.948395Z","shell.execute_reply":"2024-04-29T06:17:20.317146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\ndevice='gpu'\n#n_samples=200000\nn_est=6000\nDRY_RUN = True if sample.shape[0] == 10 else False   \nif DRY_RUN:\n    device='cpu'\n    df_base_pandas = df_base_pandas.iloc[:50000]\n    #n_samples=10000\n    n_est=600\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.319392Z","iopub.execute_input":"2024-04-29T06:17:20.319678Z","iopub.status.idle":"2024-04-29T06:17:20.330431Z","shell.execute_reply.started":"2024-04-29T06:17:20.319654Z","shell.execute_reply":"2024-04-29T06:17:20.329536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,  \n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,  \n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"device\": device, \n    \"verbose\": -1,\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.332147Z","iopub.execute_input":"2024-04-29T06:17:20.332443Z","iopub.status.idle":"2024-04-29T06:17:20.337560Z","shell.execute_reply.started":"2024-04-29T06:17:20.332420Z","shell.execute_reply":"2024-04-29T06:17:20.336667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=False) #tune this parameter","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.338536Z","iopub.execute_input":"2024-04-29T06:17:20.338784Z","iopub.status.idle":"2024-04-29T06:17:20.346455Z","shell.execute_reply.started":"2024-04-29T06:17:20.338763Z","shell.execute_reply":"2024-04-29T06:17:20.345656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_pandas_y = df_base_pandas[\"target\"]\nweeks = df_base_pandas[\"WEEK_NUM\"]\ndf_base_pandas= df_base_pandas.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.347844Z","iopub.execute_input":"2024-04-29T06:17:20.348282Z","iopub.status.idle":"2024-04-29T06:17:20.988858Z","shell.execute_reply.started":"2024-04-29T06:17:20.348249Z","shell.execute_reply":"2024-04-29T06:17:20.988055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitted_models_cat = []\nfitted_models_lgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.990007Z","iopub.execute_input":"2024-04-29T06:17:20.990388Z","iopub.status.idle":"2024-04-29T06:17:20.995395Z","shell.execute_reply.started":"2024-04-29T06:17:20.990350Z","shell.execute_reply":"2024-04-29T06:17:20.994377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_pandas[cat_cols] = df_base_pandas[cat_cols].astype(str)\n\n# df_test[cat_cols] = df_test[cat_cols].astype(str)\n\n\nfor idx_train, idx_valid in skf.split(df_base_pandas, df_base_pandas_y, groups=weeks):#\n    X_train, y_train = df_base_pandas.iloc[idx_train], df_base_pandas_y.iloc[idx_train]# \n    X_valid, y_valid = df_base_pandas.iloc[idx_valid], df_base_pandas_y.iloc[idx_valid]\n   \n    print('X_train',X_train.shape)\n   \n    train_pool = Pool(X_train, y_train,cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid,cat_features=cat_cols)\n\n    clf = CatBoostClassifier(\n    eval_metric='AUC',\n    task_type='GPU',\n    learning_rate=0.03,\n    iterations=n_est)\n\n    random_seed=3107\n    \n    clf.fit(train_pool, eval_set=val_pool,verbose=300)\n    fitted_models_cat.append(clf)\n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n    \n    \n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n    \n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(100)] )\n    \n    fitted_models_lgb.append(model)\n    y_pred_valid = model.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n\n    \n\n\n# print(\"CV AUC scores: \", cv_scores_cat)\n# print(\"Maximum CV AUC score: \", max(cv_scores_cat))\n\n\n# print(\"CV AUC scores: \", cv_scores_lgb)\n# print(\"Maximum CV AUC score: \", max(cv_scores_lgb))\n\n# print(df_base_pandas_y)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:17:20.998577Z","iopub.execute_input":"2024-04-29T06:17:20.998856Z","iopub.status.idle":"2024-04-29T06:22:55.771517Z","shell.execute_reply.started":"2024-04-29T06:17:20.998824Z","shell.execute_reply":"2024-04-29T06:22:55.770450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"CV AUC scores for catboost: \", cv_scores_cat)\nprint(\"Maximum CV AUC score for catboost: \", max(cv_scores_cat))\n\n\nprint(\"CV AUC scores for lgbm: \", cv_scores_lgb)\nprint(\"Maximum CV AUC score for lgbm: \", max(cv_scores_lgb))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:55.772808Z","iopub.execute_input":"2024-04-29T06:22:55.773153Z","iopub.status.idle":"2024-04-29T06:22:55.778945Z","shell.execute_reply.started":"2024-04-29T06:22:55.773120Z","shell.execute_reply":"2024-04-29T06:22:55.778059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fitted_models_lgb)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:55.787276Z","iopub.execute_input":"2024-04-29T06:22:55.787555Z","iopub.status.idle":"2024-04-29T06:22:55.796802Z","shell.execute_reply.started":"2024-04-29T06:22:55.787531Z","shell.execute_reply":"2024-04-29T06:22:55.795825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fitted_models_cat)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:55.798035Z","iopub.execute_input":"2024-04-29T06:22:55.798892Z","iopub.status.idle":"2024-04-29T06:22:55.803710Z","shell.execute_reply.started":"2024-04-29T06:22:55.798860Z","shell.execute_reply":"2024-04-29T06:22:55.802671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dump the model\nfrom joblib import dump\n\nprint(TRAIN_INTER)\n\nfitted_models_combined = fitted_models_lgb + fitted_models_cat\n\ndump(fitted_models_combined, f'{TRAIN_INTER}/fitted_models_combined_new.joblib')","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:55.804888Z","iopub.execute_input":"2024-04-29T06:22:55.805276Z","iopub.status.idle":"2024-04-29T06:22:56.143171Z","shell.execute_reply.started":"2024-04-29T06:22:55.805249Z","shell.execute_reply":"2024-04-29T06:22:56.142235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fitted_models_combined)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.144250Z","iopub.execute_input":"2024-04-29T06:22:56.144512Z","iopub.status.idle":"2024-04-29T06:22:56.153045Z","shell.execute_reply.started":"2024-04-29T06:22:56.144490Z","shell.execute_reply":"2024-04-29T06:22:56.152009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. TESTING PART","metadata":{}},{"cell_type":"markdown","source":"**6.1 FUNCTION - FEATURE ENGINEERING FOR TEST DATA**","metadata":{}},{"cell_type":"code","source":"\ndef feature_eng_test(df_base, depth_0, depth_1, depth_2):\n\n    # print(df_base)\n    folderPath = os.path.dirname(os.path.dirname(df_base[0][1]))\n    folderName = os.path.basename(os.path.dirname(df_base[0][1]))\n\n    outputFolderName = folderName.replace(\"inter1\",\"inter\")\n    outputFolderPath = os.path.join(folderPath,outputFolderName)\n\n    print('outputFolderPath',outputFolderPath)\n\n\n    #reading the parquet file using polars\n    df_base = pl.read_parquet(df_base[0][1])\n\n    #adding new columns to df_base\n    df_base = df_base.with_columns(\n        month_decision = pl.col(\"date_decision\").dt.month().cast(pl.Int8),\n        weekday_decision = pl.col(\"date_decision\").dt.weekday().cast(pl.Int8),\n    )\n    # print(df_base)\n\n    for i, list1 in enumerate(depth_0 + depth_1 + depth_2):\n        filename = str(list1[1])\n        \n        filename = filename.replace(\"_*\",\"\")\n        # print(filename)\n\n        if os.path.exists(filename):\n            df = pl.read_parquet(filename)\n            df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n\n    #handle dates     \n    df_base = df_base.pipe(Pipeline.handle_dates)\n\n\n    #reduce memory of data frame\n    df_base = df_base.pipe(reduce_mem_usage_polars)\n\n    #convert non numerical data to string\n    df_base = df_base.pipe(Pipeline.non_numeric_to_string)\n\n\n        \n    outputFilename = os.path.join(outputFolderPath,'joined.parquet')\n    print('outputfilename',outputFilename)\n\n    #create director if not present\n    if not os.path.exists(outputFolderPath):\n        # Create the directory\n        os.makedirs(outputFolderPath)\n\n\n    #writing the joined.parquet file to the outputFilename\n    df_base.write_parquet(outputFilename)\n\n    \n    return df_base\n\n    # print('size of df_base',df_base)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.154095Z","iopub.execute_input":"2024-04-29T06:22:56.154412Z","iopub.status.idle":"2024-04-29T06:22:56.167015Z","shell.execute_reply.started":"2024-04-29T06:22:56.154388Z","shell.execute_reply":"2024-04-29T06:22:56.166013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**6.2 FUNTION - CREATE DATA STORE FOR TEST DATA**","metadata":{}},{"cell_type":"code","source":"def create_data_store_test(path):\n    d = {\n        \"df_base\": [\n            [read_file , path / \"test_base.parquet\"],\n        ],\n\n        \"depth_0\": [\n            [read_file , path / \"test_static_cb_0.parquet\"],\n            [read_files , path / \"test_static_0_*.parquet\"],\n        ],\n\n        \"depth_1\": [\n\n            [read_files , path / \"test_applprev_1_*.parquet\", 1],\n            [read_file , path / \"test_tax_registry_a_1.parquet\", 1],\n            [read_file , path / \"test_tax_registry_b_1.parquet\", 1],\n            [read_file , path / \"test_tax_registry_c_1.parquet\", 1],\n            [read_files , path / \"test_credit_bureau_a_1_*.parquet\", 1],\n            [read_file , path / \"test_credit_bureau_b_1.parquet\", 1],\n            [read_file , path / \"test_other_1.parquet\", 1],\n            [read_file , path / \"test_person_1.parquet\", 1],\n            [read_file , path / \"test_deposit_1.parquet\", 1],\n            [read_file , path / \"test_debitcard_1.parquet\", 1],\n        ],\n\n\n        \"depth_2\": [\n            [read_file , path / \"test_credit_bureau_b_2.parquet\", 2],\n            [read_files , path / \"test_credit_bureau_a_2_*.parquet\", 2],\n            [read_file , path / \"test_applprev_2.parquet\", 2],\n            [read_file , path / \"test_person_2.parquet\", 2],\n        ]\n\n\n    }\n\n\n    return d\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.168234Z","iopub.execute_input":"2024-04-29T06:22:56.168572Z","iopub.status.idle":"2024-04-29T06:22:56.177166Z","shell.execute_reply.started":"2024-04-29T06:22:56.168542Z","shell.execute_reply":"2024-04-29T06:22:56.176229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**6.3 ACTIONS**","metadata":{}},{"cell_type":"code","source":"#load the model\nimport joblib\n\n# Load the model\nmodel = joblib.load(TRAIN_INTER / 'fitted_models_combined_new.joblib')\n\nfitted_models_combined = model\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.178328Z","iopub.execute_input":"2024-04-29T06:22:56.178620Z","iopub.status.idle":"2024-04-29T06:22:56.352330Z","shell.execute_reply.started":"2024-04-29T06:22:56.178598Z","shell.execute_reply":"2024-04-29T06:22:56.351372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VotingModel(fitted_models_combined)\nprint(len(model.estimators))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.353635Z","iopub.execute_input":"2024-04-29T06:22:56.354343Z","iopub.status.idle":"2024-04-29T06:22:56.359140Z","shell.execute_reply.started":"2024-04-29T06:22:56.354310Z","shell.execute_reply":"2024-04-29T06:22:56.358237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_used_for_training = []\nfor i in fitted_models_combined:\n    cols_used_for_training = i.feature_name_\n    break","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.360504Z","iopub.execute_input":"2024-04-29T06:22:56.360843Z","iopub.status.idle":"2024-04-29T06:22:56.368746Z","shell.execute_reply.started":"2024-04-29T06:22:56.360814Z","shell.execute_reply":"2024-04-29T06:22:56.368003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a test data store\ndata_store = create_data_store_test(TEST_DIR)\n\n# Update the intial parquet files\nupdateParquetFiles(data_store)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.369836Z","iopub.execute_input":"2024-04-29T06:22:56.370689Z","iopub.status.idle":"2024-04-29T06:22:56.919484Z","shell.execute_reply.started":"2024-04-29T06:22:56.370663Z","shell.execute_reply":"2024-04-29T06:22:56.918606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission = feature_eng_test(**create_data_store_test(TEST_INTER1))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:56.920690Z","iopub.execute_input":"2024-04-29T06:22:56.921389Z","iopub.status.idle":"2024-04-29T06:22:57.429243Z","shell.execute_reply.started":"2024-04-29T06:22:56.921354Z","shell.execute_reply":"2024-04-29T06:22:57.428470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_base_submission.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.430250Z","iopub.execute_input":"2024-04-29T06:22:57.430525Z","iopub.status.idle":"2024-04-29T06:22:57.435354Z","shell.execute_reply.started":"2024-04-29T06:22:57.430502Z","shell.execute_reply":"2024-04-29T06:22:57.434363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_base_submission.columns)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.436452Z","iopub.execute_input":"2024-04-29T06:22:57.436742Z","iopub.status.idle":"2024-04-29T06:22:57.445105Z","shell.execute_reply.started":"2024-04-29T06:22:57.436720Z","shell.execute_reply":"2024-04-29T06:22:57.444248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission = df_base_submission.select(cols_used_for_training+['case_id',])\n\nprint(df_base_submission.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.446234Z","iopub.execute_input":"2024-04-29T06:22:57.446508Z","iopub.status.idle":"2024-04-29T06:22:57.455664Z","shell.execute_reply.started":"2024-04-29T06:22:57.446484Z","shell.execute_reply":"2024-04-29T06:22:57.454727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(df_base_submission.columns)-set(df_base_pandas.columns))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.456770Z","iopub.execute_input":"2024-04-29T06:22:57.457051Z","iopub.status.idle":"2024-04-29T06:22:57.466778Z","shell.execute_reply.started":"2024-04-29T06:22:57.457028Z","shell.execute_reply":"2024-04-29T06:22:57.465831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(df_base_pandas.columns)-set(df_base_submission.columns))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.467906Z","iopub.execute_input":"2024-04-29T06:22:57.468275Z","iopub.status.idle":"2024-04-29T06:22:57.475894Z","shell.execute_reply.started":"2024-04-29T06:22:57.468245Z","shell.execute_reply":"2024-04-29T06:22:57.474958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission_pandas = df_base_submission.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.477001Z","iopub.execute_input":"2024-04-29T06:22:57.477668Z","iopub.status.idle":"2024-04-29T06:22:57.496262Z","shell.execute_reply.started":"2024-04-29T06:22:57.477642Z","shell.execute_reply":"2024-04-29T06:22:57.495498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission_pandas = reduce_mem_usage_pandas(df_base_submission_pandas)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.497649Z","iopub.execute_input":"2024-04-29T06:22:57.498365Z","iopub.status.idle":"2024-04-29T06:22:57.633164Z","shell.execute_reply.started":"2024-04-29T06:22:57.498330Z","shell.execute_reply":"2024-04-29T06:22:57.632056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_base_submission_pandas['case_id'])","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.634734Z","iopub.execute_input":"2024-04-29T06:22:57.635451Z","iopub.status.idle":"2024-04-29T06:22:57.641508Z","shell.execute_reply.started":"2024-04-29T06:22:57.635412Z","shell.execute_reply":"2024-04-29T06:22:57.640317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission_pandas[cat_cols] = df_base_submission_pandas[cat_cols].astype('str')","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.643203Z","iopub.execute_input":"2024-04-29T06:22:57.643598Z","iopub.status.idle":"2024-04-29T06:22:57.675809Z","shell.execute_reply.started":"2024-04-29T06:22:57.643558Z","shell.execute_reply":"2024-04-29T06:22:57.674768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base_submission_pandas = df_base_submission_pandas.set_index(\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.677160Z","iopub.execute_input":"2024-04-29T06:22:57.677865Z","iopub.status.idle":"2024-04-29T06:22:57.691075Z","shell.execute_reply.started":"2024-04-29T06:22:57.677829Z","shell.execute_reply":"2024-04-29T06:22:57.690392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(cols_used_for_training)- set(df_base_submission_pandas.columns))\n\nmissed_cols = set(cols_used_for_training)- set(df_base_submission_pandas.columns)\n\nfor i in list(missed_cols):\n    df_base_submission_changed = df_base_submission_changed.with_columns(\n        pl.lit(None).cast(pl.Int8).alias(i)\n    )","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.692134Z","iopub.execute_input":"2024-04-29T06:22:57.692430Z","iopub.status.idle":"2024-04-29T06:22:57.698691Z","shell.execute_reply.started":"2024-04-29T06:22:57.692406Z","shell.execute_reply":"2024-04-29T06:22:57.697773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(cat_cols) - set(cols_used_for_training))\ndf_base_submission_pandas[cat_cols] = df_base_submission_pandas[cat_cols].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.699781Z","iopub.execute_input":"2024-04-29T06:22:57.700030Z","iopub.status.idle":"2024-04-29T06:22:57.898749Z","shell.execute_reply.started":"2024-04-29T06:22:57.700009Z","shell.execute_reply":"2024-04-29T06:22:57.897989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.Series(model.predict_proba(df_base_submission_pandas)[:, 1], index=df_base_submission_pandas.index, name='score')\n\n# print(y_pred)\n\ndf_final = y_pred.to_frame()\n\nprint(df_final)\n\ndf_final.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:57.899851Z","iopub.execute_input":"2024-04-29T06:22:57.900145Z","iopub.status.idle":"2024-04-29T06:22:58.423332Z","shell.execute_reply.started":"2024-04-29T06:22:57.900119Z","shell.execute_reply":"2024-04-29T06:22:58.422314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(ROOT_INTER)\n# os.system(\"rm -rf \"+ str(ROOT_INTER))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T06:22:58.424658Z","iopub.execute_input":"2024-04-29T06:22:58.425454Z","iopub.status.idle":"2024-04-29T06:22:58.430509Z","shell.execute_reply.started":"2024-04-29T06:22:58.425425Z","shell.execute_reply":"2024-04-29T06:22:58.429422Z"},"trusted":true},"execution_count":null,"outputs":[]}]}