{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\n\nimport copy\nimport os\nimport time\nimport warnings\n\nfrom tqdm import tqdm\n\nimport catboost as cb\n\nimport matplotlib.pyplot as plt\n\nfrom scipy.optimize import minimize\n\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import KNNImputer\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:45.042070Z","iopub.execute_input":"2024-12-19T22:30:45.042520Z","iopub.status.idle":"2024-12-19T22:30:52.656234Z","shell.execute_reply.started":"2024-12-19T22:30:45.042482Z","shell.execute_reply":"2024-12-19T22:30:52.655048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.657897Z","iopub.execute_input":"2024-12-19T22:30:52.658381Z","iopub.status.idle":"2024-12-19T22:30:52.663045Z","shell.execute_reply.started":"2024-12-19T22:30:52.658350Z","shell.execute_reply":"2024-12-19T22:30:52.661957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use/\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.665413Z","iopub.execute_input":"2024-12-19T22:30:52.665699Z","iopub.status.idle":"2024-12-19T22:30:52.681492Z","shell.execute_reply.started":"2024-12-19T22:30:52.665674Z","shell.execute_reply":"2024-12-19T22:30:52.680334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Utils:\n    def __init__(self):\n        self.pciat_cols = [\n            'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05',\n            'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10',\n            'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n            'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20',\n            'PCIAT-PCIAT_Total', # 'PCIAT-Season'\n        ]\n\n        self.season_cols = [\n            'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season',\n            'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'PCIAT-Season', 'SDS-Season', 'PreInt_EduHx-Season'\n        ]\n        \n        self.features = [\n            \"PreInt_EduHx-computerinternet_hoursday\", 'SDS-SDS_Total_Raw', 'FGC-FGC_CU', 'CGAS-CGAS_Score',\n            'Basic_Demos-Age', 'Basic_Demos-Sex', 'SDS-SDS_Total_T', 'FGC-FGC_TL', 'Physical-Height',\n            'Physical-Weight', 'Physical-Waist_Circumference', 'Physical-BMI', 'Physical-Diastolic_BP', 'Physical-HeartRate',\n            'Physical-Systolic_BP', 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Time_Mins',\n            'Fitness_Endurance-Time_Sec', 'BIA-BIA_SMM', 'BIA-BIA_FFMI', 'BIA-BIA_Activity_Level_num', 'FGC-FGC_SRR'\n        ]\n        \n        self.actigraphy_original = [\n            'n_days_worn', 'ratio_days_with_error', 'avg_battery_voltage', 'ratio_non_wear', 'quarter', 'avg_wake_time',\n            'std_wake_time', 'avg_bed_time', 'std_bed_time', 'avg_sleep_duration', 'std_sleep_duration',\n            'avg_stand_sit_score_week', 'std_stand_sit_score_week', 'avg_stand_sit_score_weekend',\n            'avg_anglez_sleep', 'std_anglez_sleep', 'avg_light_delay_after_wake_up', 'avg_light_delay_before_sleep',\n            'avg_night_activity_enmo', 'std_night_activity_enmo', 'avg_day_activity_enmo_week',\n            'std_day_activity_enmo_week', 'avg_day_activity_enmo_weekend', 'avg_len_low_enmo_week',\n            'avg_len_low_enmo_weekend', 'avg_max_len_low_enmo_week', 'avg_max_len_low_enmo_weekend',\n            'avg_len_low_enmo_anglez_week', 'avg_len_low_enmo_anglez_weekend', 'avg_max_len_low_enmo_anglez_week',\n            'avg_max_len_low_enmo_anglez_weekend'\n        ]\n        \n        self.drop_act_cols = [\n            \"avg_battery_voltage\", \"quarter\", \"std_wake_time\", \"avg_stand_sit_score_week\", \"std_stand_sit_score_week\",\n            \"avg_stand_sit_score_weekend\", \"std_night_activity_enmo\", \"avg_len_low_enmo_week\", \"std_anglez_sleep\"\n        ]\n        \n        self.feat_engin = [\n            'BMI_Age', 'Internet_Hours_Age', 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE',\n            'BMR_Weight', 'DEE_Weight', 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW'\n        ]\n        \n    @property\n    def actigraphy(self):\n        actigraphy = copy.copy(self.actigraphy_original)\n        for col in drop_act_cols:\n            actigraphy.remove(col)\n        return actigraphy\n\nutils = Utils() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.683185Z","iopub.execute_input":"2024-12-19T22:30:52.683490Z","iopub.status.idle":"2024-12-19T22:30:52.700048Z","shell.execute_reply.started":"2024-12-19T22:30:52.683461Z","shell.execute_reply":"2024-12-19T22:30:52.698933Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"Before calculating features, days are split in 2 categories:\n- days with enough data points subdivided in two categories: week end weekend days. Week days are often school days, so possibly less sedentary than weekend days. \n- days with not enough data points (e.g. < 2/3 of the max number of data point a day given that the recording step is 5s and there are 86400s in a day). These days are not used to calculate the following features. The ratio of days with not enough data point on the total number of days will become a feature.\n\n----\n\n- **avg_wake_time**: average wake time.\n- **std_wake_time**: standard deviation of wake time\n- **avg_bed_time**: average bed time.\n- **std_bed_time**: standard deviation of bed time\n- **avg_sleep_duration**: average sleep duration. Bed time - wake time\n- **std_sleep_duration**: standard deviation of sleep duration\n\nTo detect wake time and bed time, I start to calculate the average enmo during a sleep period (usually data before 5:30 AM). Then, I take the first point during the morning that exceeds 1.5 times the sleep reference, and for bed time, the first point after 8 PM that drops below 1.5 times the sleep reference.\n\n- **avg_stand_sit_score_week**: knowing days and night enable to have a referential for the person's laid down position. We can measure a difference from this reference to determine whether the person stands (moderate difference) or stands up (huge difference). This feature is only for the week days.\n- **std_stand_sit_score_week**: standard deviation of the above.\n- **avg_stand_sit_score_weekend**: same, but for the weekend. I assume that there will not be enough weekend days to measure a meaningful standard deviation.\n- **avg_anglez_sleep**: average anglez during sleep time. Can be one of the referential for the above features.\n- **std_anglez_sleep**: standard deviation of the average anglez during sleep\n- **avg_light_delay_after_wake_up**: average delay between waking up and first light record. Light intensity does not really matter, but light data seem very noisy and doubts can be cast about whether lamp light could be detected.\n- **avg_light_delay_before_sleep**: average delay between last light record and bed time.\n- **avg_night_activity_enmo**: average enmo during sleep.\n- **std_night_activity_enmo**: standard deviation of enmo during sleep.\n- **avg_day_activity_enmo_week**: average enmo during the week days, when awake.\n- **std_day_activity_enmo_week**: standard deviation of enmo during the week days, when awake.\n- **avg_day_activity_enmo_weekend**: average enmo during weekend. I assume that there will not be enough weekend days to measure a meaningful standard deviation.\n- **avg_len_low_enmo_week**: in data, we can sometimes see low activity signal, with stable low enmo and stable low anglez for a few minutes, then a peak in these two, then again a low activity signal for a few minutes... This could be time spend in front of screen, with rare movement producing the peaks. This pattern is measured thanks to differences in rolling standard deviation in the enmo and/or anglez data. Here it is measured with enmo. This feature calculates the average of the length of these low activity periods during the week.\n- **avg_len_low_enmo_anglez_week**: same, but with enmo and anglez data, both showing low activities for an activity to be categorized as low.\n- **avg_len_low_enmo_weekend**: same, but for the weekend and enmo only.\n- **avg_len_low_enmo_anglez_weekend**: same, but for the weekend with enmo and anglez data.\n- **avg_max_len_low_enmo_week**: average of the maximum length of these low activity periods during the week. With enmo only.\n- **avg_max_len_low_enmo_anglez_week**: same, but with enmo and anglez data.\n- **avg_max_len_low_enmo_weekend**: same, but for the weekend and enmo data only\n- **avg_max_len_low_enmo_anglez_weekend**: same, but for the weekend, with enmo and anglez data.","metadata":{}},{"cell_type":"code","source":"\nclass FeatureEngin:\n    global_features = [\"n_days_worn\", \"ratio_days_with_error\", \"avg_battery_voltage\", \"ratio_non_wear\", \"quarter\"]\n    from_daily = [\n        \"avg_wake_time\", \"std_wake_time\",\n        \"avg_bed_time\", \"std_bed_time\",\n        \"avg_sleep_duration\", \"std_sleep_duration\",\n        \"avg_stand_sit_score_week\", \"std_stand_sit_score_week\", \"avg_stand_sit_score_weekend\",\n        \"avg_anglez_sleep\", \"std_anglez_sleep\",\n        \"avg_light_delay_after_wake_up\",\n        \"avg_light_delay_before_sleep\",\n        \"avg_night_activity_enmo\", \"std_night_activity_enmo\",\n        \"avg_day_activity_enmo_week\", \"std_day_activity_enmo_week\",\n        \"avg_day_activity_enmo_weekend\",\n        \"avg_len_low_enmo_week\", \"avg_len_low_enmo_weekend\",\n        \"avg_max_len_low_enmo_week\", \"avg_max_len_low_enmo_weekend\",\n        \"avg_len_low_enmo_anglez_week\", \"avg_len_low_enmo_anglez_weekend\",\n        \"avg_max_len_low_enmo_anglez_week\", \"avg_max_len_low_enmo_anglez_weekend\"\n    ]\n    columns = [\"anglez\", \"non-wear_flag\", \"enmo\", \"light\", \"battery_voltage\", \"time_of_day\", \"weekday\", \"quarter\",\n               \"relative_date_PCIAT\"]\n    \n    def __init__(self, id_folder: str):\n        self.df = pl.scan_parquet(data_dir + f\"/series_train.parquet/{id_folder}/part-0.parquet\").select(\n            FeatureEngin.columns).collect()\n        self.df = self.df.with_columns((pl.col(\"time_of_day\") // (10 ** 9)).alias(\"time_of_day\"))\n        _temp_diff_ = self.df[\"light\"].diff().shift(-1).fill_null(0)\n        self.df = self.df.with_columns(\n            pl.when(_temp_diff_ < 1)\n            .then(0)\n            .otherwise(pl.col(\"light\"))\n            .alias(\"light\")\n        )\n        self.features = {}\n        self.features_daily = None\n        self.days_with_error = 0\n        \n        if len(self.df[\"relative_date_PCIAT\"].unique()) >= 3:\n            self.days = list(self.df[\"relative_date_PCIAT\"].unique(maintain_order=True)[1:-2])\n        else:\n            self.days = []\n        self.weekend = []\n        \n        self.__sleep_df = None\n        self.__wake_df = None\n        self.__sleep_anglez = None\n        \n        self.plot = False\n    \n    def check_daily_data_quality(self):\n        \"\"\" Checks whether there are enough measurements points during the day to extract reliable features \"\"\"\n        df_ = self.df.clone()\n        mask = (\n                (df_[\"relative_date_PCIAT\"] > self.days[0]) &\n                (df_[\"relative_date_PCIAT\"] < self.days[-1]) &\n                (df_[\"non-wear_flag\"] == 0)\n        )\n        df_ = df_.filter(mask)  # Filter rows with the mask\n        df_grby = df_.group_by(\"relative_date_PCIAT\").agg(pl.count(\"time_of_day\").alias(\"count\")).sort(\n            by=\"relative_date_PCIAT\")\n        max_count = df_grby[\"count\"].max()\n        mask = (df_grby[\"count\"] > max_count * 0.6) & (df_grby[\"count\"] > 8000)\n        keep = df_grby.filter(mask)\n        self.days = keep[\"relative_date_PCIAT\"].to_list()\n        self.days_with_error += (~mask).sum()\n        if self.plot:\n            plt.plot(df_grby[\"relative_date_PCIAT\"].to_list(), df_grby[\"count\"].to_list())\n            plt.show()\n            plt.close()\n    \n    @staticmethod\n    def weights(df_):\n        max_time_record = df_[\"time_of_day\"][-1]\n        min_time_record = df_[\"time_of_day\"][0] - 7200\n        # Slightly wrong because max_time_record and min_time_record do not belong to the same night. However, this\n        # estimation runs faster\n        time_diff = df_[\"time_of_day\"].diff().fill_null(86400 - max_time_record + min_time_record)\n        return time_diff\n    \n    def detect_sleep_range(self, df_):\n        data = df_[[\"enmo\", \"time_of_day\"]]\n        data = data.sort(by=\"time_of_day\")\n        data_rolling = data.with_columns(\n            (pl.col(\"time_of_day\") / 3600).alias(\"time_of_day\"),\n            pl.col(\"enmo\").diff().alias(\"enmo\")\n        )\n        data_rolling = (\n            data_rolling.select([\"time_of_day\", \"enmo\"])\n            .group_by_dynamic(index_column=pl.int_range(0, pl.len()), every=\"180i\", period=\"180i\")\n            .agg(pl.col(\"time_of_day\").min().alias(\"time_of_day\"),\n                 pl.col(\"enmo\").std().alias(\"enmo\"))\n        )\n        mask = data_rolling[\"time_of_day\"] < 5.5\n        ref_sleep = data_rolling[\"enmo\"].filter(mask).max()\n        if pd.isna(ref_sleep) or ref_sleep is None:\n            ref_sleep = data_rolling[\"enmo\"].drop_nulls()[0]\n        mask = (data_rolling[\"time_of_day\"] < 12) & (data_rolling[\"enmo\"] > 1.5 * ref_sleep)\n        wake_time = data_rolling[\"time_of_day\"].filter(mask)[0]\n        mask = (data_rolling[\"time_of_day\"] > 20) & (data_rolling[\"enmo\"] > 1.5 * ref_sleep)\n        bed_time = data_rolling[\"time_of_day\"].filter(mask)[-1] + 0.25\n        return wake_time, bed_time\n\n    def create_features(self):\n        # 29 features in total\n        self.check_daily_data_quality()\n        self.find_daily_features()\n        for feat in FeatureEngin.global_features + FeatureEngin.from_daily:\n            self.features[feat] = np.nan\n        # ------------------------------------------------------------------------------\n        self.features[\"n_days_worn\"] = max(1, len(self.df[\"relative_date_PCIAT\"].unique()) - 2)\n        self.features[\"ratio_days_with_error\"] = self.days_with_error / self.features[\"n_days_worn\"]\n        self.features[\"avg_battery_voltage\"] = self.df[\"battery_voltage\"].mean()\n        self.features[\"ratio_non_wear\"] = self.df[\"non-wear_flag\"].mean()\n        self.features[\"quarter\"] = self.df[\"quarter\"].mode()[0]\n        # ------------------------------------------------------------------------------\n        if len(self.days) > 0:\n            weekend = self.features_daily[\"weekend\"].values\n            self.features[\"avg_wake_time\"] = self.features_daily[\"wake_time\"].mean()\n            self.features[\"std_wake_time\"] = self.features_daily[\"wake_time\"].std()\n            self.features[\"avg_bed_time\"] = self.features_daily[\"bed_time\"].mean()\n            self.features[\"std_bed_time\"] = self.features_daily[\"bed_time\"].std()\n            # ---\n            # sleep duration is not the length of each night of sleep, but the hours of sleep per 24 hours\n            self.features[\"avg_sleep_duration\"] = (24 - self.features_daily[\"awake_duration\"]).mean()\n            self.features[\"std_sleep_duration\"] = (24 - self.features_daily[\"awake_duration\"]).std()\n            # ---\n            self.features[\"avg_stand_sit_score_week\"] = self.features_daily.loc[~weekend, \"stand_sit_score\"].mean()\n            self.features[\"std_stand_sit_score_week\"] = self.features_daily.loc[~weekend, \"stand_sit_score\"].std()\n            self.features[\"avg_stand_sit_score_weekend\"] = self.features_daily.loc[weekend, \"stand_sit_score\"].mean()\n            # ---\n            self.features[\"avg_anglez_sleep\"] = self.features_daily[\"anglez_sleep\"].mean()\n            self.features[\"std_anglez_sleep\"] = self.features_daily[\"anglez_sleep\"].std()\n            # ---\n            self.features[\"avg_light_delay_after_wake_up\"] = self.features_daily[\"light_delay_after_wake_up\"].mean()\n            self.features[\"avg_light_delay_before_sleep\"] = self.features_daily[\"light_delay_before_sleep\"].mean()\n            self.features[\"avg_night_activity_enmo\"] = self.features_daily[\"night_activity_enmo\"].mean()\n            self.features[\"std_night_activity_enmo\"] = self.features_daily[\"night_activity_enmo\"].std()\n            # ---\n            self.features[\"avg_day_activity_enmo_week\"] = self.features_daily.loc[~weekend, \"day_activity_enmo\"].mean()\n            self.features[\"std_day_activity_enmo_week\"] = self.features_daily.loc[~weekend, \"day_activity_enmo\"].std()\n            self.features[\"avg_day_activity_enmo_weekend\"] = self.features_daily.loc[\n                weekend, \"day_activity_enmo\"].mean()\n            # ---\n            self.features[\"avg_len_low_enmo_week\"] = self.features_daily.loc[~weekend, \"avg_len_low_enmo\"].mean()\n            self.features[\"avg_max_len_low_enmo_week\"] = self.features_daily.loc[~weekend, \"max_len_low_enmo\"].mean()\n            self.features[\"avg_len_low_enmo_anglez_week\"] = self.features_daily.loc[\n                ~weekend, \"avg_len_low_enmo_anglez\"].mean()\n            self.features[\"avg_max_len_low_enmo_anglez_week\"] = self.features_daily.loc[\n                ~weekend, \"max_len_low_enmo_anglez\"].mean()\n            self.features[\"avg_len_low_enmo_weekend\"] = self.features_daily.loc[weekend, \"avg_len_low_enmo\"].mean()\n            self.features[\"avg_max_len_low_enmo_weekend\"] = self.features_daily.loc[weekend, \"max_len_low_enmo\"].mean()\n            self.features[\"avg_len_low_enmo_anglez_weekend\"] = self.features_daily.loc[\n                weekend, \"avg_len_low_enmo_anglez\"].mean()\n            self.features[\"avg_max_len_low_enmo_anglez_weekend\"] = self.features_daily.loc[\n                weekend, \"max_len_low_enmo_anglez\"].mean()\n    \n    def find_daily_features(self):\n        df_ = self.df.clone()  # Copy the DataFrame\n        # Cut day in the middle of the night (adjust relative_date_PCIAT and time_of_day)\n        mask = df_[\"time_of_day\"] < 7200\n        df_ = df_.with_columns([\n            pl.when(mask).then(df_[\"relative_date_PCIAT\"] - 1).otherwise(df_[\"relative_date_PCIAT\"]).alias(\n                \"relative_date_PCIAT\"),\n            pl.when(mask).then(df_[\"time_of_day\"] + 86400).otherwise(df_[\"time_of_day\"]).alias(\"time_of_day\")\n        ])\n        # Sort by \"relative_date_PCIAT\" and \"time_of_day\"\n        df_ = df_.sort(by=[\"relative_date_PCIAT\", \"time_of_day\"])\n        # Apply mask for filtering (non-wear_flag == 0 and relative_date_PCIAT in self.days)\n        mask = (df_[\"non-wear_flag\"] == 0) & (df_[\"relative_date_PCIAT\"].is_in(self.days))\n        df_ = df_.filter(mask).drop([\"non-wear_flag\"]).to_pandas()\n        # Reset index is unnecessary in Polars as it's not an explicit part of the DataFrame\n        # So we directly use `groupby` and apply aggregation\n        self.features_daily = df_.groupby(\"relative_date_PCIAT\").apply(self.agg_daily_features)\n        # self.features_daily = pl.from_pandas(self.features_daily)\n    \n    def agg_daily_features(self, subset):\n        try:\n            # Add weights column\n            subset = pl.from_pandas(subset)\n            subset = subset.with_columns(\n                self.weights(df_=subset).alias(\"__w__\")\n            )\n            # ------------------------------ week end --------------------------------\n            weekend = subset[\"weekday\"].mode()[0] >= 6\n            # =========================== find sleep time =============================\n            wake_time, bed_time = self.detect_sleep_range(df_=subset)\n            mask = (subset[\"time_of_day\"] < wake_time * 3600) | (subset[\"time_of_day\"] > bed_time * 3600)\n            sleep_df = subset.filter(mask)\n            wake_df = subset.filter(~mask)\n            # ------------------------------ anglez_sleep --------------------------------\n            anglez_sleep = (subset[\"anglez\"] * subset[\"__w__\"]).sum() / subset[\"__w__\"].sum()\n            # ---------------------------- stand_sit_score -------------------------------\n            value = (((wake_df[\"anglez\"] + 90) * wake_df[\"__w__\"]) - (anglez_sleep + 90)).mean()\n            stand_sit_score = value\n            # ------------------------ light_delay_after_wake_up -------------------------\n            mask = wake_df[\"light\"] == 0.0\n            if (~mask).sum() != 0:\n                value = wake_df.filter(~mask)[\"time_of_day\"].min() - wake_df[\"time_of_day\"].min()\n            else:\n                value = float(\"nan\")\n            light_delay_after_wake_up = value\n            # ------------------------ light_delay_before_sleep --------------------------\n            mask = wake_df[\"light\"] == 0.0\n            if (~mask).sum() != 0:\n                value = wake_df[\"time_of_day\"].max() - wake_df.filter(~mask)[\"time_of_day\"].max()\n            else:\n                value = float(\"nan\")\n            light_delay_before_sleep = value\n            # --------------------------- night_activity_enmo ----------------------------\n            night_activity_enmo = (sleep_df[\"enmo\"] * sleep_df[\"__w__\"]).sum() / subset[\"__w__\"].sum()\n            # ---------------------------- day_activity_enmo -----------------------------\n            day_activity_enmo = (wake_df[\"enmo\"] * wake_df[\"__w__\"]).sum() / subset[\"__w__\"].sum()\n            # -------------------------------------------------------------------------\n            (avg_len_low_enmo,\n             max_len_low_enmo,\n             avg_len_low_enmo_anglez,\n             max_len_low_enmo_anglez) = self.day_activity_classif(wake_df=wake_df)\n            \n            # Collecting results\n            output = pd.Series({\n                \"wake_time\": wake_time,\n                \"bed_time\": bed_time,\n                \"awake_duration\": bed_time - wake_time,\n                \"anglez_sleep\": anglez_sleep,\n                \"stand_sit_score\": stand_sit_score,\n                \"light_delay_after_wake_up\": light_delay_after_wake_up,\n                \"light_delay_before_sleep\": light_delay_before_sleep,\n                \"night_activity_enmo\": night_activity_enmo,\n                \"day_activity_enmo\": day_activity_enmo,\n                \"avg_len_low_enmo\": avg_len_low_enmo,\n                \"max_len_low_enmo\": max_len_low_enmo,\n                \"avg_len_low_enmo_anglez\": avg_len_low_enmo_anglez,\n                \"max_len_low_enmo_anglez\": max_len_low_enmo_anglez,\n                \"weekend\": weekend\n            })\n        except:\n            # Handling exceptions with NaN values\n            output = pd.Series({\n                \"wake_time\": float(\"nan\"),\n                \"bed_time\": float(\"nan\"),\n                \"awake_duration\": float(\"nan\"),\n                \"anglez_sleep\": float(\"nan\"),\n                \"stand_sit_score\": float(\"nan\"),\n                \"light_delay_after_wake_up\": float(\"nan\"),\n                \"light_delay_before_sleep\": float(\"nan\"),\n                \"night_activity_enmo\": float(\"nan\"),\n                \"day_activity_enmo\": float(\"nan\"),\n                \"avg_len_low_enmo\": float(\"nan\"),\n                \"max_len_low_enmo\": float(\"nan\"),\n                \"avg_len_low_enmo_anglez\": float(\"nan\"),\n                \"max_len_low_enmo_anglez\": float(\"nan\"),\n                \"weekend\": weekend\n            })\n            self.days_with_error += 1\n        \n        return output\n    \n    @staticmethod\n    def day_activity_classif(wake_df: pl.DataFrame):\n        # Normalize anglez and enmo because of later score addition\n        avg_z = wake_df[\"anglez\"].mean()\n        avg_enmo = wake_df[\"enmo\"].mean()\n        wake_df = wake_df.with_columns(\n            ((pl.col(\"anglez\") - avg_z) / pl.col(\"anglez\").std()).alias(\"z_anglez\"),\n            ((pl.col(\"enmo\") - avg_enmo) / pl.col(\"enmo\").std()).alias(\"z_enmo\")\n        )\n        # Rolling MMedAD (mean absolute deviation from the median)\n        df_rolling = (\n            wake_df.select([\"z_anglez\", \"z_enmo\", \"__w__\"])\n            .group_by_dynamic(index_column=pl.int_range(0, pl.len()), every=\"40i\", period=\"40i\")\n            .agg((pl.col(\"z_anglez\") - pl.col(\"z_anglez\").median()).abs().mean().alias(\"z_anglez\"),\n                 (pl.col(\"z_enmo\") - pl.col(\"z_enmo\").median()).abs().mean().alias(\"z_enmo\"),\n                 pl.col(\"__w__\").sum().alias(\"__w__\"))\n        )\n        df_rolling = df_rolling.with_columns(\n            (pl.col(\"z_enmo\") + pl.col(\"z_anglez\")).alias(\"z_enmo_anglez\")\n        )\n        # --- enmo ---\n        df_rolling = df_rolling.with_columns(\n            pl.when(pl.col(\"z_enmo\") >= pl.col(\"z_enmo\").mean())\n            .then(1).otherwise(0).alias(\"level_z_enmo\"),\n            pl.when(pl.col(\"z_enmo_anglez\") >= pl.col(\"z_enmo_anglez\").mean())\n            .then(1).otherwise(0).alias(\"level_z_enmo_anglez\"),\n        )\n        df_rolling = df_rolling.with_columns(\n            pl.when((pl.col(\"level_z_enmo\").diff() != 0))\n            .then(1).otherwise(0).alias(\"enmo_id\"),\n            pl.when((pl.col(\"level_z_enmo_anglez\").diff() != 0))\n            .then(1).otherwise(0).alias(\"enmo_anglez_id\")\n        )\n        df_rolling = df_rolling.with_columns(\n            pl.col(\"enmo_id\").cum_sum().fill_null(0).alias(\"enmo_id\"),\n            pl.col(\"enmo_anglez_id\").cum_sum().fill_null(0).alias(\"enmo_anglez_id\")\n        )\n        # ================================= Calculate Features ================================================\n        # ------------------ avg_len_low_enmo & max_len_low_enmo ------------------------\n        mask = df_rolling[\"level_z_enmo\"] == 0\n        w = (df_rolling[\"__w__\"].filter(mask) // 5).clip(lower_bound=1).cast(\n            pl.Int32)  # Normalize the rolling weight values\n        vc = pl.DataFrame({\"enmo_id\": df_rolling.filter(mask)[\"enmo_id\"], \"w\": w})\n        vc = vc.select(pl.exclude(\"w\").repeat_by(\"w\").explode()).to_series().value_counts()\n        avg_len_low_enmo = vc[\"count\"].mean() * 5\n        max_len_low_enmo = vc[\"count\"].max() * 5\n        # ------------- avg_len_low_enmo_anglez & max_len_low_enmo_anglez ---------------\n        mask = df_rolling[\"level_z_enmo_anglez\"] == 0\n        w = (df_rolling[\"__w__\"].filter(mask) // 5).clip(lower_bound=1).cast(pl.Int32)\n        vc = pl.DataFrame({\"enmo_anglez_id\": df_rolling.filter(mask)[\"enmo_anglez_id\"], \"w\": w})\n        vc = vc.select(pl.exclude(\"w\").repeat_by(\"w\").explode()).to_series().value_counts()\n        avg_len_low_enmo_anglez = vc[\"count\"].mean() * 5\n        max_len_low_enmo_anglez = vc[\"count\"].max() * 5\n        \n        return avg_len_low_enmo, max_len_low_enmo, avg_len_low_enmo_anglez, max_len_low_enmo_anglez\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.701314Z","iopub.execute_input":"2024-12-19T22:30:52.701740Z","iopub.status.idle":"2024-12-19T22:30:52.747532Z","shell.execute_reply.started":"2024-12-19T22:30:52.701698Z","shell.execute_reply":"2024-12-19T22:30:52.746202Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy Autoencoder","metadata":{}},{"cell_type":"code","source":"class Loader:\n    def __init__(self, file_ids):\n        self.stop = 0\n        self.file_ids = file_ids\n        self.list_ids = []\n        self.columns = [\"X\", \"Y\", \"Z\", \"anglez\", \"non-wear_flag\", \"enmo\", \"light\", \"battery_voltage\", \"time_of_day\",\n                        \"weekday\", \"relative_date_PCIAT\"]\n        self.all_data = None\n        self.operation = self.describe_operation\n        self.extra = \"\"\n        \n    def process_data(self, data):\n        # --- time to seconds ---\n        data = data.with_columns(\n            (pl.col(\"time_of_day\") // (10 ** 9)).alias(\"time_of_day\"),\n            pl.col(\"time_of_day\").cum_sum().diff().alias(\"duration\")\n        )\n        # --- light ----\n        _temp_diff_ = data[\"light\"].diff().shift(-1).fill_null(0)\n        data = data.with_columns(\n            pl.when(_temp_diff_ < 1)\n            .then(0)\n            .otherwise(pl.col(\"light\"))\n            .alias(\"light\")\n        )\n        data = self.operation(data)\n        return data\n    \n    @staticmethod\n    def describe_operation(data):\n        return data.describe().drop(\"statistic\").to_numpy().reshape(-1)\n\n    def load_and_preprocess(self, save=False, drop_cols=True):\n        list_data = []\n        for id in tqdm(self.file_ids):\n            data = pl.scan_parquet(data_dir + f\"/series_train.parquet/{id}/part-0.parquet\").select(self.columns).collect()\n            data = self.process_data(data)\n            list_data.append(data)\n        self.all_data = pd.DataFrame(list_data)\n        \n        # std column filter\n        if drop_cols:\n            std_by_col = self.all_data.std(axis=0)\n            mask = std_by_col < 0.0005\n            self.all_data = self.all_data.loc[:, self.all_data.columns[~mask]]\n    \n        print(self.all_data.shape)\n        if save:\n            self.all_data.to_csv(f\"series_data_for_autoencoder{self.extra}.csv\", index=False)\n\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim * 3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim * 3, encoding_dim * 2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim * 2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim * 2),\n            nn.ReLU(),\n            nn.Linear(input_dim * 2, input_dim * 3),\n            nn.ReLU(),\n            nn.Linear(input_dim * 3, input_dim),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, autoencoder, imputer, scaler, epochs=25, batch_size=32, verbose: bool = True):\n    df = imputer.fit_transform(df)\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    print(data_tensor.shape)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i: i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n        \n        if (epoch + 1) % 1 == 0 and verbose:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n    \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n    \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.748784Z","iopub.execute_input":"2024-12-19T22:30:52.749186Z","iopub.status.idle":"2024-12-19T22:30:52.774126Z","shell.execute_reply.started":"2024-12-19T22:30:52.749140Z","shell.execute_reply":"2024-12-19T22:30:52.773040Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Optimization","metadata":{}},{"cell_type":"code","source":"def sii_optimizer(pred, true_sii):\n    kappa_optimizer = minimize(\n        evaluate_thresholds,\n        x0=np.array([0.5, 1.5, 2.5]),\n        args=(true_sii, pred),\n        method='Nelder-Mead'\n    )\n    assert kappa_optimizer.success, \"Optimization did not converge.\"\n    \n    optimized_pred = threshold_rounder(pred, kappa_optimizer.x)\n    print(\"thresholds:\", kappa_optimizer.x)\n    print(\"Optimized Kappa ----> \", cohen_kappa_score(true_sii, optimized_pred, weights=\"quadratic\"))\n    return optimized_pred, kappa_optimizer.x\n\n\ndef sii_converter_optimizer(pred, true_sii):\n    kappa_optimizer = minimize(\n        evaluate_thresholds,\n        x0=np.array([31, 50, 79]),\n        args=(true_sii, pred),\n        method='Nelder-Mead'\n    )\n    assert kappa_optimizer.success, \"Optimization did not converge.\"\n    \n    optimized_pred = threshold_rounder(pred, kappa_optimizer.x)\n    print(\"thresholds:\", kappa_optimizer.x)\n    print(\"Optimized Kappa ----> \", cohen_kappa_score(true_sii, optimized_pred, weights=\"quadratic\"))\n    return optimized_pred, kappa_optimizer.x\n\n\ndef threshold_rounder(pred, thresholds):\n    return np.where(pred < thresholds[0], 0,\n                    np.where(pred< thresholds[1], 1,\n                             np.where(pred < thresholds[2], 2, 3)))\n\n\ndef evaluate_thresholds(thresholds, y_true, pred):\n    sii = threshold_rounder(pred, thresholds)\n    return -cohen_kappa_score(y_true, sii, weights=\"quadratic\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:30:52.775108Z","iopub.execute_input":"2024-12-19T22:30:52.775390Z","iopub.status.idle":"2024-12-19T22:30:52.793024Z","shell.execute_reply.started":"2024-12-19T22:30:52.775368Z","shell.execute_reply":"2024-12-19T22:30:52.791728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"# TRAIN SET\nfile_ids = os.listdir(data_dir + \"series_train.parquet\")\n\nlist_features = []\n\nt = time.time()\nfor i, id in enumerate(file_ids):\n    print(i, id)\n    isolated_id = id.split(\"=\")[1]\n    try:\n        fe = FeatureEngin(id_folder = id)\n        if len(fe.days) > 0:\n            fe.create_features()\n            features = fe.features\n            features[\"id\"] = isolated_id\n            list_features.append(pd.Series(features))\n        else:\n            features = pd.Series(index=utils.actigraphy_original)\n            features.loc[\"id\"] = isolated_id\n            list_features.append(features)\n    except:\n        features = pd.Series(index=utils.actigraphy_original)\n        features.loc[\"id\"] = isolated_id\n        list_features.append(features)\n\nprint(\"run time:\", round(time.time() - t),1)\n\ndf_actigraphy_fe = pd.concat(list_features, axis=1).T","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_actigraphy_fe.shape)\nprint(df_actigraphy_fe)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:44:16.538635Z","iopub.execute_input":"2024-12-19T22:44:16.539021Z","iopub.status.idle":"2024-12-19T22:44:16.554475Z","shell.execute_reply.started":"2024-12-19T22:44:16.538989Z","shell.execute_reply":"2024-12-19T22:44:16.553098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TEST SET\nfile_ids_test = os.listdir(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\nlist_features = []\n\nt = time.time()\nfor i, id in enumerate(file_ids_test):\n    print(i, id)\n    isolated_id = id.split(\"=\")[1]\n    try:\n        fe = FeatureEngin(id_folder = id)\n        if len(fe.days) > 0:\n            fe.create_features()\n            features = fe.features\n            features[\"id\"] = isolated_id\n            list_features.append(pd.Series(features))\n        else:\n            features = pd.Series(index=utils.actigraphy_original)\n            features.loc[\"id\"] = isolated_id\n            list_features.append(features)\n    except:\n        features = pd.Series(index=utils.actigraphy_original)\n        features.loc[\"id\"] = isolated_id\n        list_features.append(features)\n\nprint(\"run time:\", round(time.time() - t),1)\n\ndf_actigraphy_fe_test = pd.concat(list_features, axis=1).T","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-19T22:44:23.950415Z","iopub.execute_input":"2024-12-19T22:44:23.950756Z","iopub.status.idle":"2024-12-19T22:44:24.121355Z","shell.execute_reply.started":"2024-12-19T22:44:23.950730Z","shell.execute_reply":"2024-12-19T22:44:24.120467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_actigraphy_fe_test.shape)\nprint(df_actigraphy_fe_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:44:27.285484Z","iopub.execute_input":"2024-12-19T22:44:27.285843Z","iopub.status.idle":"2024-12-19T22:44:27.301807Z","shell.execute_reply.started":"2024-12-19T22:44:27.285814Z","shell.execute_reply":"2024-12-19T22:44:27.300656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TRAIN SET\nloader = Loader(file_ids)\nloader.load_and_preprocess(save=False)\nloader_data_train = loader.all_data\nloader_data_train.columns = loader_data_train.columns.astype(str)\n\n# TEST SET\nloader_test = Loader(file_ids_test)\nloader_test.load_and_preprocess(save=False, drop_cols=False)\nloader_data_test = loader_test.all_data\nloader_data_test.columns = loader_data_test.columns.astype(str)\nloader_data_test = loader_data_test.loc[:, loader_data_train.columns]","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-19T23:15:47.822790Z","iopub.execute_input":"2024-12-19T23:15:47.823193Z","iopub.status.idle":"2024-12-19T23:17:00.038009Z","shell.execute_reply.started":"2024-12-19T23:15:47.823160Z","shell.execute_reply":"2024-12-19T23:17:00.036915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoding_dim = 25\nepochs = 50\n\n# TRAIN SET\ndf_autoencoder = pd.concat([loader_data_train, df_actigraphy_fe], axis=1)\ndf_autoencoder = df_autoencoder.reset_index(drop=True)\nids = df_autoencoder[\"id\"]\ndf_autoencoder = df_autoencoder.drop(columns=[\"id\"])\n\nautoencoder = AutoEncoder(input_dim=df_autoencoder.shape[1], encoding_dim=encoding_dim)\nimputer = KNNImputer(n_neighbors=5)\nscaler = StandardScaler()\n    \ndf_encoded = perform_autoencoder(\n    df=df_autoencoder,\n    autoencoder=autoencoder,\n    imputer=imputer,\n    scaler=scaler,\n    epochs=epochs, \n    verbose=1\n)\n\ndf_encoded = pd.DataFrame(df_encoded)\ndf_encoded.columns = df_encoded.columns.astype(\"str\")\ndf_encoded[\"id\"] = ids\n\n\n# TEST SET\n\ndf_autoencoder = pd.concat([loader_data_test, df_actigraphy_fe_test], axis=1)\ndf_autoencoder = df_autoencoder.reset_index(drop=True)\nids = df_autoencoder[\"id\"]\ndf_autoencoder = df_autoencoder.drop(columns=[\"id\"])\n\ndf_encoded_test = imputer.transform(df_autoencoder)\ndf_encoded_test = scaler.transform(df_encoded_test)\ndf_encoded_test = torch.FloatTensor(df_encoded_test)\ndf_encoded_test = autoencoder.encoder(df_encoded_test).detach().numpy()\n\ndf_encoded_test = pd.DataFrame(df_encoded_test)\ndf_encoded_test.columns = df_encoded_test.columns.astype(\"str\")\ndf_encoded_test[\"id\"] = ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:17:11.191663Z","iopub.execute_input":"2024-12-19T23:17:11.192045Z","iopub.status.idle":"2024-12-19T23:17:17.686513Z","shell.execute_reply.started":"2024-12-19T23:17:11.192015Z","shell.execute_reply":"2024-12-19T23:17:17.684481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n    df['SDS_InternetHours'] = df['SDS-SDS_Total_T'] * df['PreInt_EduHx-computerinternet_hoursday']\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:17:41.518632Z","iopub.execute_input":"2024-12-19T23:17:41.519134Z","iopub.status.idle":"2024-12-19T23:17:41.530128Z","shell.execute_reply.started":"2024-12-19T23:17:41.519089Z","shell.execute_reply":"2024-12-19T23:17:41.528935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TRAIN SET\ndf_train = pd.read_csv(data_dir + \"train.csv\")\ndf_train = df_train.merge(df_encoded, on=\"id\", how=\"left\")\nids = df_train[\"id\"]\ndf_train = df_train.select_dtypes(\"number\")\ny = df_train[[\"sii\"] + utils.pciat_cols]\ndf_train = df_train.drop(columns=[\"sii\"])\ndf_train = df_train.drop(columns=utils.pciat_cols)\ncols = df_train.columns\nprint(len(cols), cols)\n\nimputer = KNNImputer(n_neighbors=10)\n\ndf_train = imputer.fit_transform(df_train)\ndf_train = pd.DataFrame(df_train, columns = cols)\ndf_train = feature_engineering(df_train)\n\ndf_train[\"id\"] = ids\ndf_train = pd.concat([df_train, y], axis=1)\n\n# TEST SET\ndf_test = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\nprint(df_test.shape)\ndf_test = df_test.merge(df_encoded_test, on=\"id\", how=\"left\")\nprint(df_test.shape)\nids = df_test[\"id\"]\ndf_test = df_test.select_dtypes(\"number\")\nprint(len(df_test.columns), df_test.columns)\ndf_test.columns = cols\n\ndf_test = imputer.transform(df_test)\ndf_test = pd.DataFrame(df_test, columns = cols)\ndf_test = feature_engineering(df_test)\n\ndf_test[\"id\"] = ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:22:32.885700Z","iopub.execute_input":"2024-12-19T23:22:32.886098Z","iopub.status.idle":"2024-12-19T23:22:41.486813Z","shell.execute_reply.started":"2024-12-19T23:22:32.886069Z","shell.execute_reply":"2024-12-19T23:22:41.485738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def multi_regression(\n        train: pd.DataFrame,\n        test: pd.DataFrame,\n        drop_cols: list[str] = [],\n        feature_selection: list[str] = [],\n        params: dict = {},\n        outcome: str = [\"sii\", \"PCIAT-PCIAT_Total\"],\n        verbose: int = 0,\n):\n\n    all_pred_train = []\n    all_pred_test = []\n    all_y = []\n    \n    train = train.dropna(subset=\"sii\").reset_index(drop=True)\n    \n    if len(feature_selection) > 0:\n        X_train = train[feature_selection]\n    if len(drop_cols) > 0:\n        X_train = train.drop(columns=[\"id\", \"sii\"] + utils.pciat_cols + drop_cols)\n    \n    if len(feature_selection) > 0:\n        X_test = test[feature_selection]\n    if len(drop_cols) > 0:\n        X_test = test.drop(columns=[\"id\"] + drop_cols)\n    \n    for j, outcome_ in enumerate(outcome):\n        if verbose > 0:\n            print(f\"-------- {outcome_} ---------\")\n            \n        y_train = train[outcome_]\n\n        model = cb.CatBoostRegressor(**params)\n        model.fit(X_train, y_train, verbose=100)\n        pred_train = model.predict(X_train)\n        pred_test = model.predict(X_test)\n        \n        all_pred_train.append(pred_train)\n        all_pred_test.append(pred_test)\n        all_y.append(y_train)\n\n    # sii0, threshold0 = sii_optimizer(all_pred_train[0], all_y[0])\n    # sii1, threshold1 = sii_converter_optimizer(all_pred_train[1], all_y[1])\n    # sii_train, threshold = sii_optimizer((sii0 + sii1) /2, all_y[0])\n\n    threshold0 = [0.58643777, 1.05843927, 2.65330434]\n    threshold1 = [26.34854312, 38.3922989,  87.99584238]\n    threshold = [0.525, 1.5, 2.5]\n\n    sii0 = threshold_rounder(all_pred_train[0], threshold0)\n    sii1 = threshold_rounder(all_pred_train[1], threshold1)\n    sii_train = threshold_rounder((sii0 + sii1) /2, threshold)\n    print(cohen_kappa_score(sii_train, all_y[0]))\n\n    sii0 = threshold_rounder(all_pred_test[0], threshold0)\n    sii1 = threshold_rounder(all_pred_test[1], threshold1)\n    sii_test = threshold_rounder((sii0 + sii1) /2, threshold)\n\n    sii_test = pd.DataFrame({\"sii\": sii_test})\n    sii_test[\"id\"] = test[\"id\"]\n    sii_test[[\"id\", \"sii\"]]\n    return sii_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:34:40.165806Z","iopub.execute_input":"2024-12-19T23:34:40.166197Z","iopub.status.idle":"2024-12-19T23:34:40.178335Z","shell.execute_reply.started":"2024-12-19T23:34:40.166167Z","shell.execute_reply":"2024-12-19T23:34:40.176686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"enc_columns = [col for col in df_train.columns if \"Enc\" in col]\n\nresult = multi_regression(\n    train=df_train,\n    test=df_test,\n    feature_selection=utils.features + utils.feat_engin + enc_columns,\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:34:43.249663Z","iopub.execute_input":"2024-12-19T23:34:43.250092Z","iopub.status.idle":"2024-12-19T23:34:58.636872Z","shell.execute_reply.started":"2024-12-19T23:34:43.250045Z","shell.execute_reply":"2024-12-19T23:34:58.635878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = result[[\"id\", \"sii\"]]\nprint(result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:36:02.674765Z","iopub.execute_input":"2024-12-19T23:36:02.675165Z","iopub.status.idle":"2024-12-19T23:36:02.683979Z","shell.execute_reply.started":"2024-12-19T23:36:02.675132Z","shell.execute_reply":"2024-12-19T23:36:02.681942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = result.fillna(0)\nresult.to_csv('submission.csv', index=False)\n\n# print(result['sii'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T23:36:06.571614Z","iopub.execute_input":"2024-12-19T23:36:06.572032Z","iopub.status.idle":"2024-12-19T23:36:06.579842Z","shell.execute_reply.started":"2024-12-19T23:36:06.571996Z","shell.execute_reply":"2024-12-19T23:36:06.578941Z"}},"outputs":[],"execution_count":null}]}