{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9491738,"sourceType":"datasetVersion","datasetId":5774977},{"sourceId":9496765,"sourceType":"datasetVersion","datasetId":5778993},{"sourceId":9583138,"sourceType":"datasetVersion","datasetId":5843621}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nimport glob\nimport pyarrow.parquet as pq\nSEED = 42\nn_splits = 5\nfrom pathlib import Path\nimport datetime\nimport gc\nimport sys\nfrom glob import glob\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport yaml\nfrom tqdm import tqdm\n\nfrom scipy.signal import butter, filtfilt\nfrom scipy.signal import find_peaks\n\nTRAIN_OR_TEST = \"test\"\n\npaths = glob(\n    f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/id=*/part-0.parquet\"\n)\nprint(len(paths))\n","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:06:36.246776Z","iopub.execute_input":"2024-10-24T08:06:36.247495Z","iopub.status.idle":"2024-10-24T08:06:45.132592Z","shell.execute_reply.started":"2024-10-24T08:06:36.247449Z","shell.execute_reply":"2024-10-24T08:06:45.131530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pp=pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\nbase_path=\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={}/part-0.parquet\"\nall_ids = pp[\"id\"].tolist()\nall_paths = [base_path.format(id)for id in all_ids]","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:06:45.134352Z","iopub.execute_input":"2024-10-24T08:06:45.134947Z","iopub.status.idle":"2024-10-24T08:06:45.207151Z","shell.execute_reply.started":"2024-10-24T08:06:45.134912Z","shell.execute_reply":"2024-10-24T08:06:45.206372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\"\n\nparquet_test_paths = []\n\nfor root, dirs, files in os.walk(test_series_dir):\n    for file in files:\n        if file.endswith(\".parquet\"):\n            file_path = os.path.join(root, file)\n            parquet_test_paths.append(file_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:06:45.208261Z","iopub.execute_input":"2024-10-24T08:06:45.208587Z","iopub.status.idle":"2024-10-24T08:06:45.217972Z","shell.execute_reply.started":"2024-10-24T08:06:45.208542Z","shell.execute_reply":"2024-10-24T08:06:45.217109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(df, night_offset=20):\n    return (\n        df.with_columns(\n            [\n                (pl.col(\"timestamp\").dt.year() - 2000).cast(pl.Int8).alias(\"year\"),\n                pl.col(\"timestamp\").dt.month().cast(pl.Int8).alias(\"month\"),\n                pl.col(\"timestamp\").dt.day().cast(pl.Int8).alias(\"day\"),\n                pl.col(\"timestamp\").dt.hour().cast(pl.Int8).alias(\"hour\"),\n                pl.col(\"timestamp\").dt.minute().cast(pl.Int8).alias(\"minute\"),\n                pl.col(\"timestamp\").dt.second().cast(pl.Int8).alias(\"second\"),\n                pl.col(\"timestamp\").dt.weekday().cast(pl.Int8).alias(\"weekday\"),\n            ]\n        )\n        .with_columns( \n            pl.when(pl.col(\"hour\") < night_offset)\n            .then(pl.col(\"timestamp\"))\n            .otherwise(pl.col(\"timestamp\") + pl.duration(days=1))\n            .dt.date()\n            .alias(\"night_group\"),\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"series_id\") + pl.lit(\"_\") + pl.col(\"night_group\").cast(pl.Datetime).dt.strftime(\"%Y%m%d\")\n                ).alias(\"group_id\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"timestamp\").cum_count().over(\"group_id\").alias(\"norm_step\"),\n            ]\n        )\n        .drop([\"night_group\"])\n    )\n\n\ndef transform_series(df):\n    return transform(df).with_columns(\n        [\n            (pl.col(\"enmo\") == 0).alias(\"is_enmo_clipped\"),\n        ]\n    )\n\n\ndef transform_events(df):\n    return (\n        transform(df)\n        .with_columns(\n            [\n                pl.col(\"night\").cast(pl.UInt32).alias(\"night\"),\n            ]\n        )\n        .pivot([\"step\", \"timestamp\", \"tz_offset\"], [\"series_id\", \"group_id\", \"night\"], \"event\")\n    )\n\n\ndef add_feature(\n    df,\n    day_group_col=\"group_id\",\n    term1=(5 * 60) // 5,\n    term2=(30 * 60) // 5,\n    term3=(60 * 60) // 5,\n    min_threshold=0.005,\n    max_threshold=0.04,\n    center=True,\n):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"anglez\").diff(1).abs().alias(\"anglez_diff\"),\n                pl.col(\"enmo\").diff(1).abs().alias(\"enmo_diff\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .alias(\"anglez_diff_median_5min\"),\n                pl.col(\"enmo_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .alias(\"enmo_diff_median_5min\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff_median_5min\")\n                .quantile(0.1)\n                .clip(min_threshold, max_threshold)\n                .over(day_group_col)\n                .alias(\"critical_threshold\")\n            ]\n        )\n        .with_columns([(pl.col(\"anglez_diff_median_5min\") < pl.col(\"critical_threshold\") * 15).alias(\"is_static\")])\n        .with_columns(\n            [\n                pl.col(\"is_static\").cast(pl.Int32).rolling_sum(term2, center=center).alias(\"is_static_sum_30min\"),\n            ]\n        )\n        .with_columns([(pl.col(\"is_static_sum_30min\") == ((30 * 60) // 5)).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term2 // 2).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term2 // 2)).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_sleep_block\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_sleep_block\").not_().alias(\"is_gap\")])\n        .with_columns([pl.col(\"is_gap\").cast(pl.Int32).rolling_sum(term3, center=center).alias(\"gap_length\")])\n        .with_columns([(pl.col(\"gap_length\") == term3).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term3 // 2).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term3 // 2)).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_large_gap\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_large_gap\").not_().alias(\"is_sleep_episode\")])\n        #\n        # extract longest sleep episode\n        #\n        .with_columns(\n            [\n                # extract false->true transition\n                (\n                    (\n                        pl.col(\"is_sleep_episode\")\n                        & pl.col(\"is_sleep_episode\").shift(1, fill_value=pl.lit(False)).not_()\n                    )\n                    .cum_sum()\n                    .over(\"group_id\")\n                ).alias(\"sleep_episode_id\")\n            ]\n        )\n        .with_columns(\n            [pl.col(\"is_sleep_episode\").sum().over([\"group_id\", \"sleep_episode_id\"]).alias(\"sleep_episode_length\")]\n        )\n        .with_columns([pl.col(\"sleep_episode_length\").max().over([\"group_id\"]).alias(\"max_sleep_episode_length\")])\n        .with_columns(\n            [\n                (\n                    pl.col(\"is_sleep_episode\") & (pl.col(\"sleep_episode_length\") == pl.col(\"max_sleep_episode_length\"))\n                ).alias(\"is_longest_sleep_episode\")\n            ]\n        )\n    )\n\n\nuse_columns = [\n    \"series_id\",\n    \"step\",\n    \"is_longest_sleep_episode\",\n    \"is_sleep_block\",\n    \"is_gap\",\n    \"is_large_gap\",\n    \"is_sleep_episode\",\n    \"is_static\",\n]\n\n# for path in tqdm(paths):\nfor path in tqdm(parquet_test_paths):\n    \n    sdf = pl.read_parquet(path)\n\n    # dummy timestamp\n    sdf = sdf.with_columns((pl.col(\"time_of_day\") == 0).cast(pl.Int32).cum_sum().alias(\"day_offset\"))\n    sdf = sdf.with_columns(\n        (\n            datetime.datetime(2020, 1, 1)\n            + (pl.col(\"day_offset\") * 86400_000_000 + pl.col(\"time_of_day\") / 1000).cast(pl.Duration(\"us\"))\n        ).alias(\"timestamp\")\n    )\n\n    sdf = sdf.with_columns(pl.lit(path.split(\"/\")[-2]).alias(\"series_id\"))\n    sdf = sdf.sort(\"step\")\n    sdf = transform_series(sdf)\n    sdf = add_feature(sdf)\n    sdf = sdf[use_columns].fill_null(False)\n\n    sidf = path.split(\"/\")[-2]\n    save_path = f\"/kaggle/working/heuristic_features/{sidf}.parquet\"\n    os.makedirs(os.path.dirname(save_path), exist_ok=True)\n    sdf.write_parquet(save_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:06:45.219956Z","iopub.execute_input":"2024-10-24T08:06:45.220243Z","iopub.status.idle":"2024-10-24T08:06:46.009811Z","shell.execute_reply.started":"2024-10-24T08:06:45.220213Z","shell.execute_reply":"2024-10-24T08:06:46.008254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    sys.path.append(\"/kaggle/input/modelfile/cmi-2023-src-20240927T055310Z-001/cmi-2023-src\")\n    from consts import ANGLEZ_MEAN, ANGLEZ_STD, ENMO_MEAN, ENMO_STD\n    from torch_models.dataset import ZzzPatchDataset\n    from torch_models.models import ZzzConv1dGRUModel, ZzzTransformerGRUModel, ZzzWaveGRUModel\n\n    from utils.feature_contena import Features\n    from utils.lightning_utils import MyLightningDataModule, MyLightningModule\n    from utils.set_seed import seed_base_torch\n    from utils.torch_template import EnsembleModel","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:06:46.010967Z","iopub.execute_input":"2024-10-24T08:06:46.011246Z","iopub.status.idle":"2024-10-24T08:07:00.888423Z","shell.execute_reply.started":"2024-10-24T08:06:46.011217Z","shell.execute_reply":"2024-10-24T08:07:00.887625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = \"patch_transformer_gru\"\n\nPACKAGE_DIR = Path(\"/kaggle/input/modelfile/cmi-2023-src-20240927T055310Z-001/cmi-2023-src\")\nCFG = yaml.safe_load(open(PACKAGE_DIR / \"config.yaml\", \"r\"))\nBLOCK_SIZE = CFG[MODEL_NAME][\"execution\"][\"block_size\"]\n\nCFG[\"output_dir\"] = f\"/kaggle/input/cmi-2023-output/{CFG[MODEL_NAME]['execution']['best_exp_id']}\"\n\nseed_base_torch(CFG[\"env\"][\"seed\"])\n\nDEVICE = \"cuda\"\n\nfiles = glob(\n    f\"/kaggle/input/child-mind-institute-problematic-internet-use/series_{TRAIN_OR_TEST}.parquet/id=*/part-0.parquet\"\n)\n\nvalid_files = []\n\nfor file_path in files:\n    df = pd.read_parquet(file_path)\n    null_counts = df.isnull().sum().sum()\n    if null_counts == 0:\n        valid_files.append(file_path)\n\nprint(null_counts)\n\n# files= all_use_path\nfeatures = Features()\nfeatures.add_num_features([\"anglez\", \"enmo\"])\nfeatures.add_num_features([\"anglez_diff\", \"enmo_diff\"])\nfeatures.add_num_features([\"same_count\"])\nfeatures.add_num_features([\"large_diff_count\"])\nfeatures.add_num_features([\"same_count_shift_plus\", \"same_count_shift_minus\"])\nfeatures.add_num_features([\"is_longest_sleep_episode\", \"is_sleep_block\"])\n\n# transformer + gru\nmodel = ZzzTransformerGRUModel(\n    max_len=BLOCK_SIZE // CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    input_numerical_size=len(features.all_features()) * CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    **CFG[MODEL_NAME][\"params\"],\n)\ntrn_models = [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(\"/kaggle/input/modelfile/cmi-2023-output-20240927T055308Z-001/cmi-2023-output/exp_160\", f\"logs/best_model_fold{fold}.ckpt\"),\n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(valid_files) > 100 else 1)\n]\n\nmodels = trn_models\nmodel = EnsembleModel(models).to(DEVICE)\nmodel.eval()\n\nall_oof_dfs = []\nfor file in tqdm(valid_files):\n    # load file\n    df = pd.read_parquet(file)\n    if len(df) < BLOCK_SIZE:\n        print(f\"File {file} skipped due to size: {len(df)}\")\n        continue\n    time_of_days = df[\"time_of_day\"].values\n\n    # same_count\n    DAY_STEPS = 12 * 60 * 24\n    n_days = int(len(df) // DAY_STEPS) + 1\n    df[\"same_count\"] = 0\n    for day in range(-n_days, n_days + 1):\n        if day == 0:\n            continue\n        df[\"_anglez_diff\"] = df[\"anglez\"].diff(DAY_STEPS * day)\n        df[\"_anglez_diff\"] = df[\"_anglez_diff\"].fillna(1)\n        df[\"same_count\"] += (df[\"_anglez_diff\"] == 0).astype(int)\n    df[\"same_count\"] = (df[\"same_count\"].clip(0, 5) - 2.5) / 2.5\n\n    SHIFT_STEPS = 12 * 60 * 6  # 6h\n    df[\"same_count_shift_plus\"] = df[\"same_count\"].shift(SHIFT_STEPS).fillna(1.0).astype(np.float16)\n    df[\"same_count_shift_minus\"] = df[\"same_count\"].shift(-SHIFT_STEPS).fillna(1.0).astype(np.float16)\n\n    # features\n    df[\"anglez_diffabs\"] = df[\"anglez\"].diff().abs().fillna(0)\n    df[\"large_diff\"] = (df[\"anglez_diffabs\"] > 5).astype(int)\n    df[\"large_diff_count\"] = df[\"large_diff\"].rolling(10, center=True).mean().fillna(0)\n    df[\"large_diff_count\"] = (df[\"large_diff_count\"] - 0.5) * 2\n\n    # normalize\n    df[\"anglez\"] = (df[\"anglez\"] - ANGLEZ_MEAN) / ANGLEZ_STD\n    df[\"enmo\"] = (df[\"enmo\"] - ENMO_MEAN) / ENMO_STD\n    df[\"anglez_diff\"] = df[\"anglez\"].diff().fillna(0)\n    df[\"enmo_diff\"] = df[\"enmo\"].diff().fillna(0)\n\n    # heuristic_features by @bilzard\n    sid = file.split(\"/\")[-2]\n    df[\"series_id\"] = sid\n    path = f\"/kaggle/working/heuristic_features/{sid}.parquet\"\n    hdf = pd.read_parquet(path)\n    df = pd.concat([df, hdf.drop(columns=[\"series_id\", \"step\"])], axis=1)\n    df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] = df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] * 2 - 1\n\n    # split\n    dfs = []\n    df = df.sort_values(\"step\").reset_index(drop=True)\n    for start in range(0, len(df), BLOCK_SIZE // 8):\n        end = start + BLOCK_SIZE\n        if end > len(df):\n            end = len(df) - len(df) % CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n            start = end - BLOCK_SIZE\n            assert start >= 0\n        assert df.iloc[start][\"step\"] % CFG[MODEL_NAME][\"execution\"][\"patch_size\"] == 0\n        dfs.append(df.iloc[start:end])\n    gc.collect()\n\n    # inference\n    train_dataset = ZzzPatchDataset(\n        dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    )\n    valid_dataset = ZzzPatchDataset(\n        dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    )\n    data_module = MyLightningDataModule(train_dataset, valid_dataset, batch_size=64)\n    preds = []\n    with torch.no_grad():\n        for X in data_module.val_dataloader():\n            pred = torch.sigmoid(model(X.to(\"cuda\"))).detach().cpu().numpy() * 10\n            preds.append(pred)\n\n    oof_dfs = []\n    for pred, df in zip(np.vstack(preds), dfs):\n        df = df.iloc[\n            CFG[MODEL_NAME][\"execution\"][\"patch_size\"] // 2 : len(df) : CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n        ].reset_index(drop=True)\n        df[[\"wakeup_oof\", \"onset_oof\"]] = pred\n        oof_dfs.append(df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]])\n\n    oof_df = pd.concat(oof_dfs)\n    oof_df = oof_df.groupby([\"series_id\", \"step\"]).mean().reset_index().sort_values([\"series_id\", \"step\"])\n    oof_df = oof_df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]]\n    oof_df[\"step\"] = oof_df[\"step\"].astype(int)\n\n    del preds, oof_dfs\n    gc.collect()\n\n    train = oof_df.reset_index(drop=True)\n    train[\"time_of_day\"] = time_of_days[\n        CFG[MODEL_NAME][\"execution\"][\"patch_size\"] // 2 :: CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n    ][: len(train)]\n    all_oof_dfs.append(train[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\", \"time_of_day\"]])\n    # del dfs, df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:07:00.889950Z","iopub.execute_input":"2024-10-24T08:07:00.890708Z","iopub.status.idle":"2024-10-24T08:07:06.957937Z","shell.execute_reply.started":"2024-10-24T08:07:00.890672Z","shell.execute_reply":"2024-10-24T08:07:06.956995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# savea\nfor df in tqdm(all_oof_dfs):\n    save_path = f\"/kaggle/working/features/sleep_detection/{df['series_id'].iloc[0]}.parquet\"\n    os.makedirs(os.path.dirname(save_path), exist_ok=True)\n    df.to_parquet(save_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:07:06.959222Z","iopub.execute_input":"2024-10-24T08:07:06.959536Z","iopub.status.idle":"2024-10-24T08:07:07.006843Z","shell.execute_reply.started":"2024-10-24T08:07:06.959502Z","shell.execute_reply":"2024-10-24T08:07:07.005587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_dataframe(df,id_, threshold=2):\n    \n    file_peak_times = []\n    \n    if 'time_of_day' in df.columns:\n        time_of_day_ns = df['time_of_day'].values\n    else:\n        print(f\"ファイル: {file} に 'time_of_day' カラムがありません。\")\n\n    wakeup_values = df['wakeup_oof'].values\n    wakeup_peaks, _ = find_peaks(wakeup_values)  # ピークを検出\n\n    threshold = 2  # 0付近の値の閾値を設定\n    valid_wakeup_peaks = [peak for peak in wakeup_peaks if wakeup_values[peak] > threshold]\n\n    wakeup_peak_times_ns = []\n    if len(valid_wakeup_peaks) > 0:\n        for peak in valid_wakeup_peaks:\n            peak_time_ns = time_of_day_ns[peak]  # ピークに対応するナノ秒の時間を取得\n            wakeup_peak_times_ns.append(peak_time_ns)\n\n    if 'onset_oof' in df.columns:\n        onset_values = df['onset_oof'].values\n        onset_peaks, _ = find_peaks(onset_values)\n        valid_onset_peaks = [peak for peak in onset_peaks if onset_values[peak] > threshold]\n\n        onset_peak_times_ns = []\n        if len(valid_onset_peaks) > 0:\n            for peak in valid_onset_peaks:\n                onset_time_ns = time_of_day_ns[peak]  # ピークに対応するナノ秒の時間を取得\n                onset_peak_times_ns.append(onset_time_ns)\n\n        if onset_peak_times_ns:\n            average_onset_oof_ns = int(np.mean(onset_peak_times_ns))  # ナノ秒での平均を計算し、intに丸める\n            average_onset_oof_time = pd.to_timedelta(average_onset_oof_ns, unit='ns').round('s')\n        else:\n            average_onset_oof_time = None\n            average_onset_oof_ns = None  # 新しく追加\n\n    else:\n        average_onset_oof_time = None\n        average_onset_oof_ns = None  # 新しく追加\n\n    # non-wear_flag が0でない割合を計算\n    if 'non-wear_flag' in df.columns:\n        non_wear_ratio = np.sum(df['non-wear_flag'] != 0) / len(df)\n    else:\n        non_wear_ratio = None\n        \n    if wakeup_peak_times_ns:\n        average_wakeup_ns = int(np.mean(wakeup_peak_times_ns))\n        file_peak_times.append((id_,average_wakeup_ns, average_onset_oof_ns, non_wear_ratio))\n    else:\n        file_peak_times.append((id_, None, average_onset_oof_ns, non_wear_ratio))\n\n    result_df = pd.DataFrame(file_peak_times, columns=['id','average_wakeup_time_ns', 'average_onset_time_ns', 'non_wear_ratio'])\n\n\n    return result_df\n","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:07:07.008527Z","iopub.execute_input":"2024-10-24T08:07:07.009065Z","iopub.status.idle":"2024-10-24T08:07:07.021975Z","shell.execute_reply.started":"2024-10-24T08:07:07.009005Z","shell.execute_reply":"2024-10-24T08:07:07.021004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_single_dataframe(sleep_df, base_df):\n    \n        current_day = 1\n        day_list = []\n\n        # sleep_dfのtime_of_dayカラムの最大値を取得\n        max_time_of_day = sleep_df['time_of_day'].max()\n\n        # sleep_dfに「day」列を作成\n        for time_of_day in sleep_df['time_of_day']:\n            # time_of_dayの最大値が来たらcurrent_dayを更新\n            if time_of_day == max_time_of_day:\n                current_day += 1\n            day_list.append(current_day)\n\n        sleep_df['day'] = day_list\n\n        # base_dfに「day」列を作成\n        base_df['day'] = base_df['weekday'].ne(base_df['weekday'].shift()).cumsum()\n\n        # base_dfに「wakeup_oof」「onset_oof」列を作成\n        base_df['wakeup_oof'] = 0\n        base_df['onset_oof'] = 0\n\n        for day in sleep_df['day'].unique():\n            sleep_day = sleep_df[sleep_df['day'] == day]\n            base_day = base_df[base_df['day'] == day]\n\n            # base_day が空かどうかを確認\n            if base_day.empty:\n                print(f\"Warning: base_day is empty for day {day}\")\n                continue\n\n            for idx, row in sleep_day.iterrows():\n                closest_idx = (base_day['time_of_day'] - row['time_of_day']).abs().idxmin()\n\n                base_df.at[closest_idx, 'wakeup_oof'] = row['wakeup_oof']\n                base_df.at[closest_idx, 'onset_oof'] = row['onset_oof']\n        \n                \n        return base_df\n\n\n\n# ディレクトリパスを定義\nbase_path_template = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/id={}/part-0.parquet\"\nsleep_path_template = \"/kaggle/working/features/sleep_detection/id={}.parquet\"\n\n\nbase_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/\"\nall_base_files = [Path(file) for file in valid_files]\nall_ids = [file.parent.name.split('id=')[-1] for file in all_base_files]\n\nall_processed_data = []\nall_test_results = []\n\nfor id_ in all_ids:\n    base_file_path = base_path_template.format(id_)\n    sleep_file_path = sleep_path_template.format(id_)\n    \n    base_df = pq.read_table(base_file_path).to_pandas()\n    sleep_df = pq.read_table(sleep_file_path).to_pandas()\n\n    processed_base_df = process_single_dataframe(sleep_df, base_df)\n    result_df=process_dataframe(processed_base_df,id_)\n\n    all_processed_data.append(processed_base_df)\n    all_test_results.append(result_df)\nall_test_results=pd.concat(all_test_results, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:07:07.023011Z","iopub.execute_input":"2024-10-24T08:07:07.023295Z","iopub.status.idle":"2024-10-24T08:07:20.343738Z","shell.execute_reply.started":"2024-10-24T08:07:07.023259Z","shell.execute_reply":"2024-10-24T08:07:20.342499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_train_results = []\n\nfiles = []\nfor i in range(1, 11):\n    files.extend(Path(f'/kaggle/input/convertfiles/convert{i}').rglob('*.parquet'))\n\nfor file in tqdm(files):\n    file_id = file.stem  # .parquetを除いたファイル名を取得\n    # id=の部分を取得する場合は次の行のように修正\n    # file_id = file.parent.name.split('id=')[-1]\n\n    df = pd.read_parquet(file)\n    result_df = process_dataframe(df, file_id)\n    all_train_results.append(result_df)\n\nall_train_results = pd.concat(all_train_results, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:07:20.348906Z","iopub.execute_input":"2024-10-24T08:07:20.349468Z","iopub.status.idle":"2024-10-24T08:08:59.279868Z","shell.execute_reply.started":"2024-10-24T08:07:20.349343Z","shell.execute_reply":"2024-10-24T08:08:59.278860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# train = train.drop('id', axis=1)\n# test = test.drop('id', axis=1)\n\nfeaturesCols = ['id','Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n\"\"\"This Mapping Works Fine For me I also Check Each Values in Train and test Using Logic. There no Data Lekage.\"\"\"\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:08:59.281657Z","iopub.execute_input":"2024-10-24T08:08:59.281982Z","iopub.status.idle":"2024-10-24T08:10:26.101068Z","shell.execute_reply.started":"2024-10-24T08:08:59.281948Z","shell.execute_reply":"2024-10-24T08:10:26.100039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_added = pd.merge(train, all_train_results, on='id', how='left')\ntrain_added=train_added.drop(\"id\",axis=True)\ntrain_added[\"average_wakeup_time_ns\"]=pd.to_numeric(train_added['average_wakeup_time_ns'], errors='coerce')\ntrain_added[\"average_onset_time_ns\"]=pd.to_numeric(train_added['average_onset_time_ns'], errors='coerce')\ntrain=train_added","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:26.102210Z","iopub.execute_input":"2024-10-24T08:10:26.102550Z","iopub.status.idle":"2024-10-24T08:10:26.119211Z","shell.execute_reply.started":"2024-10-24T08:10:26.102514Z","shell.execute_reply":"2024-10-24T08:10:26.118209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.merge(test, all_test_results, on='id', how='left')\ntest = test.drop('id', axis=1)\ntest[\"average_wakeup_time_ns\"]=pd.to_numeric(test['average_wakeup_time_ns'], errors='coerce')\ntest[\"average_onset_time_ns\"]=pd.to_numeric(test['average_onset_time_ns'], errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:26.120622Z","iopub.execute_input":"2024-10-24T08:10:26.121874Z","iopub.status.idle":"2024-10-24T08:10:26.132411Z","shell.execute_reply.started":"2024-10-24T08:10:26.121838Z","shell.execute_reply":"2024-10-24T08:10:26.131633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission,model","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:26.134171Z","iopub.execute_input":"2024-10-24T08:10:26.134500Z","iopub.status.idle":"2024-10-24T08:10:26.150678Z","shell.execute_reply.started":"2024-10-24T08:10:26.134466Z","shell.execute_reply":"2024-10-24T08:10:26.149550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nParams7 = {'learning_rate': 0.03884249148676395, 'max_depth': 12, 'num_leaves': 413, 'min_data_in_leaf': 14,\n           'feature_fraction': 0.7987976913702801, 'bagging_fraction': 0.7602261703576205, 'bagging_freq': 2, \n           'lambda_l1': 4.735462555910575, 'lambda_l2': 4.735028557007343e-06} # CV : 0.4094 | LB : 0.471\n\nLight = lgb.LGBMRegressor(**Params7,random_state=SEED, verbose=-1,n_estimators=200)\nSubmission,model = TrainML(Light,test)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:26.152000Z","iopub.execute_input":"2024-10-24T08:10:26.152362Z","iopub.status.idle":"2024-10-24T08:10:33.747184Z","shell.execute_reply.started":"2024-10-24T08:10:26.152319Z","shell.execute_reply":"2024-10-24T08:10:33.746201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfeature_importance_df = pd.DataFrame({\n    'Feature': model.booster_.feature_name(),\n    'Importance': model.booster_.feature_importance(importance_type='gain')\n})\n\nfeature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n\nplt.figure(figsize=(20, 40))\nsns.barplot(x='Importance', y='Feature', data=feature_importance_df.head(100)) \nplt.title(\"Top Feature Importance\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:33.748409Z","iopub.execute_input":"2024-10-24T08:10:33.748760Z","iopub.status.idle":"2024-10-24T08:10:35.414439Z","shell.execute_reply.started":"2024-10-24T08:10:33.748726Z","shell.execute_reply":"2024-10-24T08:10:35.413531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Submission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:35.415747Z","iopub.execute_input":"2024-10-24T08:10:35.416059Z","iopub.status.idle":"2024-10-24T08:10:35.426462Z","shell.execute_reply.started":"2024-10-24T08:10:35.416026Z","shell.execute_reply":"2024-10-24T08:10:35.425475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Submission","metadata":{"execution":{"iopub.status.busy":"2024-10-24T08:10:35.427720Z","iopub.execute_input":"2024-10-24T08:10:35.428088Z","iopub.status.idle":"2024-10-24T08:10:35.441387Z","shell.execute_reply.started":"2024-10-24T08:10:35.428042Z","shell.execute_reply":"2024-10-24T08:10:35.440347Z"},"trusted":true},"execution_count":null,"outputs":[]}]}