{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9448132,"sourceType":"datasetVersion","datasetId":5742470},{"sourceId":9448136,"sourceType":"datasetVersion","datasetId":5742473}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I run a model from last year’s competition to detect sleep periods based on anglez and enmo. \n\nAfter detecting these sleep periods, I generated some features relevant to them.\n\n\nNotes:\n- I ran one of my models from last year. For more details, please visit [here](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/459597)\n- Approximately 60% of the data has timesteps of 5 seconds, while the remaining data does not. This notebook does not address this noise.\n- Time zone information is not included; I expect that it has already been corrected by host.","metadata":{}},{"cell_type":"code","source":"import datetime\nimport gc\nimport os\nimport sys\nfrom glob import glob\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nfrom glob import glob\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nimport yaml\nfrom tqdm import tqdm\n\nTRAIN_OR_TEST = \"train\"\n\npaths = glob(\n    f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/id=*/part-0.parquet\"\n)\nprint(len(paths))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-21T09:56:54.749761Z","iopub.execute_input":"2024-09-21T09:56:54.750737Z","iopub.status.idle":"2024-09-21T09:56:55.604759Z","shell.execute_reply.started":"2024-09-21T09:56:54.750685Z","shell.execute_reply":"2024-09-21T09:56:55.603766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess\n\nFirst, prepare the features used in my sleep detection model. Please refer to the implementation by [@tatamikenn](https://www.kaggle.com/tatamikenn) [here](https://www.kaggle.com/code/tatamikenn/sleep-hdcza-a-pure-heuristic-approach-lb-0-447).","metadata":{}},{"cell_type":"code","source":"def transform(df, night_offset=20):\n    return (\n        df.with_columns(\n            [\n                (pl.col(\"timestamp\").dt.year() - 2000).cast(pl.Int8).alias(\"year\"),\n                pl.col(\"timestamp\").dt.month().cast(pl.Int8).alias(\"month\"),\n                pl.col(\"timestamp\").dt.day().cast(pl.Int8).alias(\"day\"),\n                pl.col(\"timestamp\").dt.hour().cast(pl.Int8).alias(\"hour\"),\n                pl.col(\"timestamp\").dt.minute().cast(pl.Int8).alias(\"minute\"),\n                pl.col(\"timestamp\").dt.second().cast(pl.Int8).alias(\"second\"),\n                pl.col(\"timestamp\").dt.weekday().cast(pl.Int8).alias(\"weekday\"),\n            ]\n        )\n        .with_columns( \n            pl.when(pl.col(\"hour\") < night_offset)\n            .then(pl.col(\"timestamp\"))\n            .otherwise(pl.col(\"timestamp\") + pl.duration(days=1))\n            .dt.date()\n            .alias(\"night_group\"),\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"series_id\") + pl.lit(\"_\") + pl.col(\"night_group\").cast(pl.Datetime).dt.strftime(\"%Y%m%d\")\n                ).alias(\"group_id\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"timestamp\").cum_count().over(\"group_id\").alias(\"norm_step\"),\n            ]\n        )\n        .drop([\"night_group\"])\n    )\n\n\ndef transform_series(df):\n    return transform(df).with_columns(\n        [\n            (pl.col(\"enmo\") == 0).alias(\"is_enmo_clipped\"),\n        ]\n    )\n\n\ndef transform_events(df):\n    return (\n        transform(df)\n        .with_columns(\n            [\n                pl.col(\"night\").cast(pl.UInt32).alias(\"night\"),\n            ]\n        )\n        .pivot([\"step\", \"timestamp\", \"tz_offset\"], [\"series_id\", \"group_id\", \"night\"], \"event\")\n    )\n\n\ndef add_feature(\n    df,\n    day_group_col=\"group_id\",\n    term1=(5 * 60) // 5,\n    term2=(30 * 60) // 5,\n    term3=(60 * 60) // 5,\n    min_threshold=0.005,\n    max_threshold=0.04,\n    center=True,\n):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"anglez\").diff(1).abs().alias(\"anglez_diff\"),\n                pl.col(\"enmo\").diff(1).abs().alias(\"enmo_diff\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .alias(\"anglez_diff_median_5min\"),\n                pl.col(\"enmo_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .alias(\"enmo_diff_median_5min\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff_median_5min\")\n                .quantile(0.1)\n                .clip(min_threshold, max_threshold)\n                .over(day_group_col)\n                .alias(\"critical_threshold\")\n            ]\n        )\n        .with_columns([(pl.col(\"anglez_diff_median_5min\") < pl.col(\"critical_threshold\") * 15).alias(\"is_static\")])\n        .with_columns(\n            [\n                pl.col(\"is_static\").cast(pl.Int32).rolling_sum(term2, center=center).alias(\"is_static_sum_30min\"),\n            ]\n        )\n        .with_columns([(pl.col(\"is_static_sum_30min\") == ((30 * 60) // 5)).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term2 // 2).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term2 // 2)).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_sleep_block\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_sleep_block\").not_().alias(\"is_gap\")])\n        .with_columns([pl.col(\"is_gap\").cast(pl.Int32).rolling_sum(term3, center=center).alias(\"gap_length\")])\n        .with_columns([(pl.col(\"gap_length\") == term3).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term3 // 2).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term3 // 2)).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_large_gap\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_large_gap\").not_().alias(\"is_sleep_episode\")])\n        #\n        # extract longest sleep episode\n        #\n        .with_columns(\n            [\n                # extract false->true transition\n                (\n                    (\n                        pl.col(\"is_sleep_episode\")\n                        & pl.col(\"is_sleep_episode\").shift(1, fill_value=pl.lit(False)).not_()\n                    )\n                    .cum_sum()\n                    .over(\"group_id\")\n                ).alias(\"sleep_episode_id\")\n            ]\n        )\n        .with_columns(\n            [pl.col(\"is_sleep_episode\").sum().over([\"group_id\", \"sleep_episode_id\"]).alias(\"sleep_episode_length\")]\n        )\n        .with_columns([pl.col(\"sleep_episode_length\").max().over([\"group_id\"]).alias(\"max_sleep_episode_length\")])\n        .with_columns(\n            [\n                (\n                    pl.col(\"is_sleep_episode\") & (pl.col(\"sleep_episode_length\") == pl.col(\"max_sleep_episode_length\"))\n                ).alias(\"is_longest_sleep_episode\")\n            ]\n        )\n    )\n\n\nuse_columns = [\n    \"series_id\",\n    \"step\",\n    \"is_longest_sleep_episode\",\n    \"is_sleep_block\",\n    \"is_gap\",\n    \"is_large_gap\",\n    \"is_sleep_episode\",\n    \"is_static\",\n]\n\nfor path in tqdm(paths):\n    sdf = pl.read_parquet(path)\n\n    # dummy timestamp\n    sdf = sdf.with_columns((pl.col(\"time_of_day\") == 0).cast(pl.Int32).cum_sum().alias(\"day_offset\"))\n    sdf = sdf.with_columns(\n        (\n            datetime.datetime(2020, 1, 1)\n            + (pl.col(\"day_offset\") * 86400_000_000 + pl.col(\"time_of_day\") / 1000).cast(pl.Duration(\"us\"))\n        ).alias(\"timestamp\")\n    )\n\n    sdf = sdf.with_columns(pl.lit(path.split(\"/\")[-2]).alias(\"series_id\"))\n    sdf = sdf.sort(\"step\")\n    sdf = transform_series(sdf)\n    sdf = add_feature(sdf)\n    sdf = sdf[use_columns].fill_null(False)\n\n    sidf = path.split(\"/\")[-2]\n    save_path = f\"/kaggle/working/heuristic_features/{sidf}.parquet\"\n    os.makedirs(os.path.dirname(save_path), exist_ok=True)\n    sdf.write_parquet(save_path)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T09:56:55.606902Z","iopub.execute_input":"2024-09-21T09:56:55.607237Z","iopub.status.idle":"2024-09-21T10:01:49.004631Z","shell.execute_reply.started":"2024-09-21T09:56:55.607203Z","shell.execute_reply":"2024-09-21T10:01:49.003719Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sleep Detection","metadata":{}},{"cell_type":"code","source":"if True:\n    sys.path.append(\"/kaggle/input/cmi-2023-src\")\n    from consts import ANGLEZ_MEAN, ANGLEZ_STD, ENMO_MEAN, ENMO_STD\n    from torch_models.dataset import ZzzPatchDataset\n    from torch_models.models import ZzzConv1dGRUModel, ZzzTransformerGRUModel, ZzzWaveGRUModel\n\n    from utils.feature_contena import Features\n    from utils.lightning_utils import MyLightningDataModule, MyLightningModule\n    from utils.set_seed import seed_base_torch\n    from utils.torch_template import EnsembleModel","metadata":{"execution":{"iopub.status.busy":"2024-09-21T10:01:49.005705Z","iopub.execute_input":"2024-09-21T10:01:49.006010Z","iopub.status.idle":"2024-09-21T10:01:49.011544Z","shell.execute_reply.started":"2024-09-21T10:01:49.005978Z","shell.execute_reply":"2024-09-21T10:01:49.010652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = \"patch_transformer_gru\"\n\nPACKAGE_DIR = Path(\"/kaggle/input/cmi-2023-src\")\nCFG = yaml.safe_load(open(PACKAGE_DIR / \"config.yaml\", \"r\"))\nBLOCK_SIZE = CFG[MODEL_NAME][\"execution\"][\"block_size\"]\n\nCFG[\"output_dir\"] = f\"/kaggle/input/cmi-2023-output/{CFG[MODEL_NAME]['execution']['best_exp_id']}\"\n\nseed_base_torch(CFG[\"env\"][\"seed\"])\n\nDEVICE = \"cuda\"\n\nfiles = glob(\n    f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/id=*/part-0.parquet\"\n)\n\nfeatures = Features()\nfeatures.add_num_features([\"anglez\", \"enmo\"])\nfeatures.add_num_features([\"anglez_diff\", \"enmo_diff\"])\nfeatures.add_num_features([\"same_count\"])\nfeatures.add_num_features([\"large_diff_count\"])\nfeatures.add_num_features([\"same_count_shift_plus\", \"same_count_shift_minus\"])\nfeatures.add_num_features([\"is_longest_sleep_episode\", \"is_sleep_block\"])\n\n# transformer + gru\nmodel = ZzzTransformerGRUModel(\n    max_len=BLOCK_SIZE // CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    input_numerical_size=len(features.all_features()) * CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    **CFG[MODEL_NAME][\"params\"],\n)\ntrn_models = [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(\"/kaggle/input/cmi-2023-output/exp_160\", f\"logs/best_model_fold{fold}.ckpt\"),\n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(files) > 100 else 1)\n]\n\nmodels = trn_models\nmodel = EnsembleModel(models).to(DEVICE)\nmodel.eval()\n\nall_oof_dfs = []\nfor file in tqdm(files):\n    # load file\n    df = pd.read_parquet(file)\n    if len(df) < BLOCK_SIZE:\n        continue\n    time_of_days = df[\"time_of_day\"].values\n\n    # same_count\n    DAY_STEPS = 12 * 60 * 24\n    n_days = int(len(df) // DAY_STEPS) + 1\n    df[\"same_count\"] = 0\n    for day in range(-n_days, n_days + 1):\n        if day == 0:\n            continue\n        df[\"_anglez_diff\"] = df[\"anglez\"].diff(DAY_STEPS * day)\n        df[\"_anglez_diff\"] = df[\"_anglez_diff\"].fillna(1)\n        df[\"same_count\"] += (df[\"_anglez_diff\"] == 0).astype(int)\n    df[\"same_count\"] = (df[\"same_count\"].clip(0, 5) - 2.5) / 2.5\n\n    SHIFT_STEPS = 12 * 60 * 6  # 6h\n    df[\"same_count_shift_plus\"] = df[\"same_count\"].shift(SHIFT_STEPS).fillna(1.0).astype(np.float16)\n    df[\"same_count_shift_minus\"] = df[\"same_count\"].shift(-SHIFT_STEPS).fillna(1.0).astype(np.float16)\n\n    # features\n    df[\"anglez_diffabs\"] = df[\"anglez\"].diff().abs().fillna(0)\n    df[\"large_diff\"] = (df[\"anglez_diffabs\"] > 5).astype(int)\n    df[\"large_diff_count\"] = df[\"large_diff\"].rolling(10, center=True).mean().fillna(0)\n    df[\"large_diff_count\"] = (df[\"large_diff_count\"] - 0.5) * 2\n\n    # normalize\n    df[\"anglez\"] = (df[\"anglez\"] - ANGLEZ_MEAN) / ANGLEZ_STD\n    df[\"enmo\"] = (df[\"enmo\"] - ENMO_MEAN) / ENMO_STD\n    df[\"anglez_diff\"] = df[\"anglez\"].diff().fillna(0)\n    df[\"enmo_diff\"] = df[\"enmo\"].diff().fillna(0)\n\n    # heuristic_features by @bilzard\n    sid = file.split(\"/\")[-2]\n    df[\"series_id\"] = sid\n    path = f\"/kaggle/working/heuristic_features/{sid}.parquet\"\n    hdf = pd.read_parquet(path)\n    df = pd.concat([df, hdf.drop(columns=[\"series_id\", \"step\"])], axis=1)\n    df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] = df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] * 2 - 1\n\n    # split\n    dfs = []\n    df = df.sort_values(\"step\").reset_index(drop=True)\n    for start in range(0, len(df), BLOCK_SIZE // 8):\n        end = start + BLOCK_SIZE\n        if end > len(df):\n            end = len(df) - len(df) % CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n            start = end - BLOCK_SIZE\n            assert start >= 0\n        assert df.iloc[start][\"step\"] % CFG[MODEL_NAME][\"execution\"][\"patch_size\"] == 0\n        dfs.append(df.iloc[start:end])\n    gc.collect()\n\n    # inference\n    train_dataset = ZzzPatchDataset(\n        dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    )\n    valid_dataset = ZzzPatchDataset(\n        dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    )\n    data_module = MyLightningDataModule(train_dataset, valid_dataset, batch_size=64)\n    preds = []\n    with torch.no_grad():\n        for X in data_module.val_dataloader():\n            pred = torch.sigmoid(model(X.to(\"cuda\"))).detach().cpu().numpy() * 10\n            preds.append(pred)\n\n    oof_dfs = []\n    for pred, df in zip(np.vstack(preds), dfs):\n        df = df.iloc[\n            CFG[MODEL_NAME][\"execution\"][\"patch_size\"] // 2 : len(df) : CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n        ].reset_index(drop=True)\n        df[[\"wakeup_oof\", \"onset_oof\"]] = pred\n        oof_dfs.append(df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]])\n\n    oof_df = pd.concat(oof_dfs)\n    oof_df = oof_df.groupby([\"series_id\", \"step\"]).mean().reset_index().sort_values([\"series_id\", \"step\"])\n    oof_df = oof_df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]]\n    oof_df[\"step\"] = oof_df[\"step\"].astype(int)\n\n    del preds, oof_dfs\n    gc.collect()\n\n    train = oof_df.reset_index(drop=True)\n    train[\"time_of_day\"] = time_of_days[\n        CFG[MODEL_NAME][\"execution\"][\"patch_size\"] // 2 :: CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    ][: len(train)]\n    all_oof_dfs.append(train[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\", \"time_of_day\"]])\n    # del dfs, df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T10:01:49.013951Z","iopub.execute_input":"2024-09-21T10:01:49.014246Z","iopub.status.idle":"2024-09-21T11:33:35.282229Z","shell.execute_reply.started":"2024-09-21T10:01:49.014214Z","shell.execute_reply":"2024-09-21T11:33:35.281220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save\nfor df in tqdm(all_oof_dfs):\n    save_path = f\"/kaggle/working/features/sleep_detection/{df['series_id'].iloc[0]}.parquet\"\n    os.makedirs(os.path.dirname(save_path), exist_ok=True)\n    df.to_parquet(save_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T11:33:35.283777Z","iopub.execute_input":"2024-09-21T11:33:35.284296Z","iopub.status.idle":"2024-09-21T11:33:49.199765Z","shell.execute_reply.started":"2024-09-21T11:33:35.284247Z","shell.execute_reply":"2024-09-21T11:33:49.198771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check!\nsample_file = \"/kaggle/working/features/sleep_detection/id=0d279d77.parquet\"\n\ndf = pl.read_parquet(sample_file)\ndf = df.with_columns(pl.col(\"step\").cast(pl.UInt32)).drop(\"time_of_day\")\nsid = df[\"series_id\"][0]\n\nsensor_df = pl.read_parquet(\n    f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/{sid}/part-0.parquet\"\n).with_columns((pl.col(\"time_of_day\") == 0).cum_sum().alias(\"day\"))\n\nsensor_df = sensor_df.join(df, on=\"step\", how=\"left\").with_columns(\n    pl.col(\"onset_oof\").interpolate(),\n    pl.col(\"wakeup_oof\").interpolate(),\n)\n\nfor (day, ), day_df in sensor_df.group_by(\"day\", maintain_order=True):\n    fig, axs = plt.subplots(3, 1, figsize=(20, 3))\n    times = np.linspace(0, 24, len(day_df))\n    axs[0].plot(times, day_df[\"enmo\"])\n    axs[0].set_ylabel(\"enmo\")\n    axs[1].plot(times, day_df[\"anglez\"])\n    axs[1].set_ylabel(\"anglez\")\n    axs[2].plot(times, day_df[\"onset_oof\"])\n    axs[2].plot(times, day_df[\"wakeup_oof\"])\n    axs[2].set_ylabel(\"oof\")\n    axs[2].set_ylim(0, 10)\n    plt.tight_layout()\n    plt.show()\n    if day > 5:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-09-21T11:33:49.200972Z","iopub.execute_input":"2024-09-21T11:33:49.201285Z","iopub.status.idle":"2024-09-21T11:33:52.890347Z","shell.execute_reply.started":"2024-09-21T11:33:49.201252Z","shell.execute_reply":"2024-09-21T11:33:52.889377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"time_of_day_max = 86400000000000\nall_files = sorted(glob(\"/kaggle/working/features/sleep_detection/*.parquet\"))\nlen(all_files)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T11:33:52.891683Z","iopub.execute_input":"2024-09-21T11:33:52.892595Z","iopub.status.idle":"2024-09-21T11:33:52.904625Z","shell.execute_reply.started":"2024-09-21T11:33:52.892551Z","shell.execute_reply":"2024-09-21T11:33:52.903634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = []\ndebug_count = 0\nfor file in tqdm(all_files):\n    df = pl.read_parquet(file)\n    df = df.with_columns(pl.col(\"step\").cast(pl.UInt32)).drop(\"time_of_day\")\n    sid = df[\"series_id\"][0]\n\n    sensor_df = pl.read_parquet(\n        f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/{sid}/part-0.parquet\"\n    ).with_columns((pl.col(\"time_of_day\") == 0).cum_sum().alias(\"day\"))\n\n    feature = {\n        \"id\": sid,\n        \"length\": df.shape[0],\n        \"day\": sensor_df[\"relative_date_PCIAT\"].max() - sensor_df[\"relative_date_PCIAT\"].min(),\n    }\n\n    # skip if time step is not 5sec\n    diffs = sensor_df[\"time_of_day\"].diff().drop_nulls().unique()\n    if set(diffs) != set([-86395000000000, 5000000000]):\n        features.append(feature)\n        continue\n\n    sensor_df = (\n        sensor_df.join(df, on=\"step\", how=\"left\")\n        .sort(\"step\")\n        .with_columns(\n            pl.col(\"onset_oof\").interpolate(),\n            pl.col(\"wakeup_oof\").interpolate(),\n        )\n    )\n\n    # onset = 15:00~3:00, wakeup = 3:00~15:00\n    onset_start = time_of_day_max / 24 * 15  # 15:00\n    onset_end = time_of_day_max / 24 * 3  # 3:00\n    sensor_df = sensor_df.with_columns(\n        ((pl.col(\"time_of_day\") > onset_start) | (pl.col(\"time_of_day\") < onset_end)).alias(\"onset_duration\"),\n    ).with_columns(\n        pl.col(\"onset_duration\").cast(pl.Int32).diff().fill_null(0).abs().cum_sum().alias(\"onset_wakeup_duration\")\n    )\n\n    # get sleep period\n    sleep_info = []\n    for _, df in sensor_df.group_by(\"onset_wakeup_duration\", maintain_order=True):\n        is_onset = df[\"onset_duration\"][0]\n        if is_onset:\n            max_idx = df[\"onset_oof\"].arg_max()\n            if max_idx is None:\n                continue\n            max_score = df[\"onset_oof\"][max_idx]\n            step = df[\"step\"][max_idx]\n\n            # date\n            start_time = df[\"time_of_day\"][0] / time_of_day_max * 24\n            if start_time >= 15:\n                day = df[\"day\"][0]\n                week_day = df[\"weekday\"][0]\n            else:\n                day = df[\"day\"][0] - 1\n                week_day = df[\"weekday\"][0] - 1\n                if week_day == 0:\n                    week_day = 7\n        else:\n            max_idx = df[\"wakeup_oof\"].arg_max()\n            if max_idx is None:\n                continue\n            max_score = df[\"wakeup_oof\"][max_idx]\n            step = df[\"step\"][max_idx]\n\n            # date\n            start_time = df[\"time_of_day\"][0] / time_of_day_max * 24\n            day = df[\"day\"][0] - 1\n            week_day = df[\"weekday\"][0] - 1\n\n        info = {\n            \"day\": day,\n            \"weekday\": week_day,\n            \"type\": \"onset\" if is_onset else \"wakeup\",\n            \"step\": step,\n            \"max_score\": max_score,\n            \"time\": df[\"time_of_day\"][max_idx] / time_of_day_max * 24,\n        }\n        sleep_info.append(info)\n    sleep_df = pl.DataFrame(sleep_info)\n\n    # merge\n    sleep_df = (\n        sleep_df.filter(pl.col(\"type\") == \"onset\")\n        .drop(\"type\")\n        .rename(\n            {\n                \"max_score\": \"onset_score\",\n                \"step\": \"onset_step\",\n                \"time\": \"onset_time\",\n            }\n        )\n        .join(\n            sleep_df.filter(pl.col(\"type\") == \"wakeup\")\n            .drop([\"type\", \"weekday\"])\n            .rename(\n                {\n                    \"max_score\": \"wakeup_score\",\n                    \"step\": \"wakeup_step\",\n                    \"time\": \"wakeup_time\",\n                }\n            ),\n            on=\"day\",\n        )\n    ).select(\n        [\"day\", \"weekday\", \"onset_time\", \"wakeup_time\", \"onset_step\", \"wakeup_step\", \"onset_score\", \"wakeup_score\"]\n    )\n\n    # feature engineering\n    sleep_lengths = []  # wakeup - onset\n    sleep_enmo_mean = []  \n    sleep_enmo_std = []  \n    sleep_light_mean = []\n    sleep_light_std = [] \n    for i in range(len(sleep_df)):\n        # sleep period\n        start = sleep_df[\"onset_step\"][i]\n        end = sleep_df[\"wakeup_step\"][i]\n        if sleep_df[\"onset_score\"][i] < 1 or sleep_df[\"wakeup_score\"][i] < 1:\n            sleep_lengths.append(np.nan)\n            sleep_enmo_mean.append(np.nan)\n            sleep_enmo_std.append(np.nan)\n            sleep_light_mean.append(np.nan)\n            sleep_light_std.append(np.nan)\n            continue\n\n        # sleep length\n        length = end - start\n        sleep_lengths.append(length * 5 / 60 / 60)  # hour\n\n        # enmo\n        enmo_mean = sensor_df[\"enmo\"][start:end].mean()\n        enmo_std = sensor_df[\"enmo\"][start:end].std()\n        sleep_enmo_mean.append(enmo_mean)\n        sleep_enmo_std.append(enmo_std)\n\n        # light\n        light_mean = sensor_df[\"light\"][start:end].mean()\n        light_std = sensor_df[\"light\"][start:end].std()\n        sleep_light_mean.append(light_mean)\n        sleep_light_std.append(light_std)\n        \n    sleep_df = sleep_df.with_columns(\n        pl.DataFrame(\n            {\n                \"sleep_length\": sleep_lengths,\n                \"sleep_enmo_mean\": sleep_enmo_mean,\n                \"sleep_enmo_std\": sleep_enmo_std,\n                \"sleep_light_mean\": sleep_light_mean,\n                \"sleep_light_std\": sleep_light_std,\n            }\n        )\n    )\n    \n    # leave only high confidence periods\n    sleep_df = sleep_df.filter((pl.col(\"wakeup_score\") > 1) & (pl.col(\"onset_score\") > 1))\n    if debug_count < 3:\n        display(sleep_df.head())\n    debug_count += 1\n        \n\n    # agg\n    feature.update(\n        {\n            \"sleep_measurement_count\": sleep_df.shape[0],\n            \"sleep_length_mean\": sleep_df[\"sleep_length\"].mean(),\n            \"sleep_length_std\": sleep_df[\"sleep_length\"].std(),\n            \"sleep_start_mean\": sleep_df[\"onset_time\"].mean(),\n            \"sleep_start_std\": sleep_df[\"onset_time\"].std(),\n            \"sleep_end_mean\": sleep_df[\"wakeup_time\"].mean(),\n            \"sleep_end_std\": sleep_df[\"wakeup_time\"].std(),\n            \"sleep_enmo_mean_mean\": sleep_df[\"sleep_enmo_mean\"].mean(),\n            \"sleep_enmo_mean_std\": sleep_df[\"sleep_enmo_mean\"].std(),\n            \"sleep_enmo_std_mean\": sleep_df[\"sleep_enmo_std\"].mean(),\n            \"sleep_enmo_std_std\": sleep_df[\"sleep_enmo_std\"].std(),\n            \"sleep_light_mean_mean\": sleep_df[\"sleep_light_mean\"].mean(),\n            \"sleep_light_mean_std\": sleep_df[\"sleep_light_mean\"].std(),\n            \"sleep_light_std_mean\": sleep_df[\"sleep_light_std\"].mean(),\n            \"sleep_light_std_std\": sleep_df[\"sleep_light_std\"].std(),\n        }\n    )\n    features.append(feature)\n    \nfeature_df = pl.DataFrame(features).with_columns(pl.col(\"id\").str.slice(3, 8))\nfeature_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T11:33:52.906294Z","iopub.execute_input":"2024-09-21T11:33:52.906975Z","iopub.status.idle":"2024-09-21T11:34:41.708335Z","shell.execute_reply.started":"2024-09-21T11:33:52.906912Z","shell.execute_reply":"2024-09-21T11:34:41.707327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_df.write_csv(\"/kaggle/working/features/sleep_features.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-09-21T11:34:41.709658Z","iopub.execute_input":"2024-09-21T11:34:41.710425Z","iopub.status.idle":"2024-09-21T11:34:41.722680Z","shell.execute_reply.started":"2024-09-21T11:34:41.710389Z","shell.execute_reply":"2024-09-21T11:34:41.721670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}