{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"dbcb0307-fecd-497d-ac01-16a7ef544895","cell_type":"code","source":"import glob\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nDATA_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:18.445985Z","iopub.execute_input":"2024-11-21T13:14:18.447054Z","iopub.status.idle":"2024-11-21T13:14:21.890449Z","shell.execute_reply.started":"2024-11-21T13:14:18.447011Z","shell.execute_reply":"2024-11-21T13:14:21.889372Z"}},"outputs":[],"execution_count":null},{"id":"79e5d94f-28b6-4500-a247-36fc7ee3576b","cell_type":"code","source":"id_series_data = pd.read_parquet(DATA_PATH + 'series_train.parquet/id=00115b9f/part-0.parquet') # 某一个参与者的数据\nid_series_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:28.624723Z","iopub.execute_input":"2024-11-21T13:14:28.625256Z","iopub.status.idle":"2024-11-21T13:14:28.852962Z","shell.execute_reply.started":"2024-11-21T13:14:28.625222Z","shell.execute_reply":"2024-11-21T13:14:28.851601Z"}},"outputs":[],"execution_count":null},{"id":"7a31cfb2-a574-404e-b110-0c1c0ac39cf0","cell_type":"markdown","source":"在HBN研究期间，一些参与者被要求佩戴加速度计，最长可达30天，以便在家中进行日常活动时进行持续监测。数据文件 `series_{train|test}.parquet/id={id}` 包含用于训练的数据，按参与者的唯一标识符id进行分区。每个系列是单个参与者的加速度计数据的连续记录，涵盖多天的时间段。具体字段说明如下：\n\n- **id**：与 `train/test.csv` 中的id字段对应的患者标识符。\n- **step**：每个观察值在系列中的整数时间步。\n- **X, Y, Z**：手腕佩戴的手表沿每个标准轴所经历的加速度测量值，以g为单位。\n- **enmo**：根据wristpy包计算的ENMO（Euclidean Norm Minus One），是所有加速度计信号（沿x、y和z轴，以g力为单位）的欧几里得范数减去1，负值四舍五入为零。零值表示无运动的时间段。虽然在这个空间中没有标准的加速度测量，但这是几种常用计算特征之一。\n- **anglez**：根据wristpy包计算的Angle-Z，是从各个加速度计组件派生出的一个指标，表示手臂与水平面之间的角度。\n- **non-wear_flag**：一个标志（0：手表正在佩戴，1：手表未佩戴），用于确定手表被移除的时间段，基于GGIR定义，该定义使用加速度计数据的标准差和范围。\n- **light**：环境光的测量值，以lux为单位。\n- **battery_voltage**：电池电压的测量值，以mV为单位。\n- **time_of_day**：表示数据采样的5秒窗口开始的时间，格式为%H:%M:%S.%9f。\n- **weekday**：星期几的编码整数，1表示星期一，7表示星期日。\n- **quarter**：年份的季度，整数范围为1至4。\n- **relative_date_PCIAT**：自PCIAT测试实施以来的天数（整数），负值表示在测试实施之前收集了活动监测数据。 ","metadata":{}},{"id":"a64ec2f4-0a00-4721-bc8f-1fb7a87be858","cell_type":"code","source":"id_series_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:30.528971Z","iopub.execute_input":"2024-11-21T13:14:30.529371Z","iopub.status.idle":"2024-11-21T13:14:30.570473Z","shell.execute_reply.started":"2024-11-21T13:14:30.529334Z","shell.execute_reply":"2024-11-21T13:14:30.569183Z"}},"outputs":[],"execution_count":null},{"id":"afe65483-5b88-40e0-b938-96615cef1e79","cell_type":"markdown","source":"```python\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n```","metadata":{}},{"id":"c3ece76b-d0a1-41cc-853c-f764814b57af","cell_type":"code","source":"# 不定长的时间序列 -》 统计的特征提取 -》 相同维度向量\nid_series_data.drop('step', axis=1).describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:32.297192Z","iopub.execute_input":"2024-11-21T13:14:32.297719Z","iopub.status.idle":"2024-11-21T13:14:32.370602Z","shell.execute_reply.started":"2024-11-21T13:14:32.297665Z","shell.execute_reply":"2024-11-21T13:14:32.369271Z"}},"outputs":[],"execution_count":null},{"id":"ddaffe84-bdb2-4c46-bc35-e5247fbc0a9d","cell_type":"code","source":"columns_to_process = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage']\n\n# Function to compute global statistics\ndef compute_global_features(df, columns):\n    features = {}\n    for col in columns:\n        features[f'{col}_mean'] = df[col].mean()\n        features[f'{col}_std'] = df[col].std()\n        features[f'{col}_min'] = df[col].min()\n        features[f'{col}_max'] = df[col].max()\n        features[f'{col}_25th'] = df[col].quantile(0.25)\n        features[f'{col}_50th'] = df[col].median()\n        features[f'{col}_75th'] = df[col].quantile(0.75)\n        features[f'{col}_range'] = df[col].max() - df[col].min()\n        features[f'{col}_skew'] = df[col].skew()\n        features[f'{col}_kurtosis'] = df[col].kurtosis()\n    return pd.Series(features)\n\n# Compute the feature vector for the entire time series\nglobal_features = compute_global_features(id_series_data, columns_to_process)\n\n# Add cyclic features based on time_of_day and weekday\nid_series_data['hour'] = pd.to_datetime(id_series_data['time_of_day']).dt.hour\nid_series_data['is_weekend'] = id_series_data['weekday'].isin([6, 7]).astype(int)\n\ncyclic_features = {\n    'hour_mean': id_series_data['hour'].mean(),\n    'hour_std': id_series_data['hour'].std(),\n    'is_weekend_mean': id_series_data['is_weekend'].mean(),\n}\n\n# Combine everything into a single feature vector\nfinal_feature_vector = pd.concat([global_features, pd.Series(cyclic_features)])","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:33.367522Z","iopub.execute_input":"2024-11-21T13:14:33.368585Z","iopub.status.idle":"2024-11-21T13:14:33.446335Z","shell.execute_reply.started":"2024-11-21T13:14:33.368538Z","shell.execute_reply":"2024-11-21T13:14:33.445123Z"}},"outputs":[],"execution_count":null},{"id":"6431703c-4166-496d-b9dd-6feac36883c7","cell_type":"code","source":"final_feature_vector","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:34.418069Z","iopub.execute_input":"2024-11-21T13:14:34.418630Z","iopub.status.idle":"2024-11-21T13:14:34.430156Z","shell.execute_reply.started":"2024-11-21T13:14:34.418577Z","shell.execute_reply":"2024-11-21T13:14:34.428573Z"}},"outputs":[],"execution_count":null},{"id":"8ab91fac-5adc-4075-9f34-def9e0d30538","cell_type":"code","source":"id_series_data['hour'] = pd.to_datetime(id_series_data['time_of_day']).dt.hour\nid_series_data['is_day'] = ((id_series_data['hour'] >= 6) & (id_series_data['hour'] < 18)).astype(int)\nid_series_data['is_weekend'] = id_series_data['weekday'].isin([6, 7]).astype(int)\n\ncolumns_to_process = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage']\n\ndef compute_stats(df, columns):\n    stats = {}\n    for col in columns:\n        stats[f'{col}_mean'] = df[col].mean()\n        stats[f'{col}_std'] = df[col].std()\n        stats[f'{col}_min'] = df[col].min()\n        stats[f'{col}_max'] = df[col].max()\n        stats[f'{col}_median'] = df[col].median()\n    return pd.Series(stats)\n\n# Compute statistics for day and night\nday_stats = compute_stats(id_series_data[id_series_data['is_day'] == 1], columns_to_process)\nnight_stats = compute_stats(id_series_data[id_series_data['is_day'] == 0], columns_to_process)\n\n# Compute statistics for weekday and weekend\nweekday_stats = compute_stats(id_series_data[id_series_data['is_weekend'] == 0], columns_to_process)\nweekend_stats = compute_stats(id_series_data[id_series_data['is_weekend'] == 1], columns_to_process)\n\n# Calculate differences\nday_night_diff = (day_stats - night_stats).add_suffix('_day_night_diff')\nweekday_weekend_diff = (weekday_stats - weekend_stats).add_suffix('_weekday_weekend_diff')\n\n# Combine all features into a single vector\nfinal_features = pd.concat([day_night_diff, weekday_weekend_diff])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:34.925177Z","iopub.execute_input":"2024-11-21T13:14:34.925567Z","iopub.status.idle":"2024-11-21T13:14:34.994907Z","shell.execute_reply.started":"2024-11-21T13:14:34.925532Z","shell.execute_reply":"2024-11-21T13:14:34.993720Z"}},"outputs":[],"execution_count":null},{"id":"c3463b26-7ccb-4af7-93fd-9041798678cd","cell_type":"code","source":"final_features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:35.682177Z","iopub.execute_input":"2024-11-21T13:14:35.682563Z","iopub.status.idle":"2024-11-21T13:14:35.691386Z","shell.execute_reply.started":"2024-11-21T13:14:35.682529Z","shell.execute_reply":"2024-11-21T13:14:35.690219Z"}},"outputs":[],"execution_count":null},{"id":"35a0eb38-03ad-4f6a-875b-99c991bbdc6c","cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n# 有监督的自编码器\nclass MLP_Autoencoder(nn.Module):\n    def __init__(self):\n        super(MLP_Autoencoder, self).__init__()\n        \n        # Encoder\n        self.encoder = nn.Sequential(\n            nn.Linear(100, 50), \n            nn.ReLU(),\n            nn.Linear(50, 20) \n        )\n        \n        # Decoder\n        self.decoder = nn.Sequential(\n            nn.Linear(20, 50), \n            nn.ReLU(),\n            nn.Linear(50, 100)  \n        )\n        \n        # Classification header\n        self.cls_head = nn.Linear(20, 1)\n\n    def forward(self, x):\n        # Encoder\n        latent = self.encoder(x)\n        \n        # Decoder\n        reconstructed = self.decoder(latent)\n        \n        # Classification\n        cls_output = self.cls_head(latent)\n        \n        return reconstructed, cls_output\n\n    def extract_features(self, x):\n        return self.encoder(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:36.713431Z","iopub.execute_input":"2024-11-21T13:14:36.713860Z","iopub.status.idle":"2024-11-21T13:14:40.520374Z","shell.execute_reply.started":"2024-11-21T13:14:36.713823Z","shell.execute_reply":"2024-11-21T13:14:40.519085Z"}},"outputs":[],"execution_count":null},{"id":"19884955-0982-42a7-8f3a-36092ec4973a","cell_type":"code","source":"model = MLP_Autoencoder()\n\nreconstruction_loss_fn = nn.MSELoss()\nclassification_loss_fn = nn.BCEWithLogitsLoss()\n\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\nx = torch.rand(32, 100)\ny_true = torch.randint(0, 2, (32, 1)).float() \n\nreconstructed, cls_output = model(x)\n\nreconstruction_loss = reconstruction_loss_fn(reconstructed, x)\nclassification_loss = classification_loss_fn(cls_output, y_true)\n\ntotal_loss = reconstruction_loss + classification_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:40.522324Z","iopub.execute_input":"2024-11-21T13:14:40.522929Z","iopub.status.idle":"2024-11-21T13:14:41.743994Z","shell.execute_reply.started":"2024-11-21T13:14:40.522890Z","shell.execute_reply":"2024-11-21T13:14:41.742463Z"}},"outputs":[],"execution_count":null},{"id":"102fa7bf-1656-47a2-bcc7-3280977a36e4","cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass CNN1DRegression(nn.Module):\n    def __init__(self):\n        super(CNN1DRegression, self).__init__()\n        self.conv1 = nn.Conv1d(in_channels=100, out_channels=64, kernel_size=3, stride=1, padding=1)\n        self.conv2 = nn.Conv1d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1)\n        self.conv3 = nn.Conv1d(in_channels=32, out_channels=16, kernel_size=3, stride=1, padding=1)\n        \n        self.fc1 = nn.Linear(2560, 128)\n        self.fc2 = nn.Linear(128, 1)\n        self.relu = nn.ReLU()\n        self.pool = nn.MaxPool1d(kernel_size=2, stride=2)\n    \n    def forward(self, x):\n        # x shape: (batch_size, time_steps, features)\n        x = x.permute(0, 2, 1)\n        x = self.relu(self.conv1(x))\n        x = self.pool(x)\n        x = self.relu(self.conv2(x))\n        x = self.pool(x)\n        x = self.relu(self.conv3(x))\n        x = self.pool(x)\n        \n        x = x.view(x.size(0), -1)\n        x = self.relu(self.fc1(x))\n        x = self.fc2(x)\n        \n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:42.686280Z","iopub.execute_input":"2024-11-21T13:14:42.687324Z","iopub.status.idle":"2024-11-21T13:14:42.697143Z","shell.execute_reply.started":"2024-11-21T13:14:42.687280Z","shell.execute_reply":"2024-11-21T13:14:42.695904Z"}},"outputs":[],"execution_count":null},{"id":"c5d11a3b-484b-4347-8bc6-e308350041c1","cell_type":"code","source":"model = CNN1DRegression()\ninput_data = torch.randn(32, 1280, 100)\noutput = model(input_data)\noutput.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:43.188251Z","iopub.execute_input":"2024-11-21T13:14:43.188721Z","iopub.status.idle":"2024-11-21T13:14:43.462122Z","shell.execute_reply.started":"2024-11-21T13:14:43.188672Z","shell.execute_reply":"2024-11-21T13:14:43.460705Z"}},"outputs":[],"execution_count":null},{"id":"c1050366-b53c-4c5b-9608-37530bd5e6dc","cell_type":"code","source":"class CNN1DEncoderDecoder(nn.Module):\n    def __init__(self, input_dim, latent_dim):\n        super(CNN1DEncoderDecoder, self).__init__()\n        \n        # Encoder\n        self.encoder_conv1 = nn.Conv1d(in_channels=input_dim, out_channels=64, kernel_size=3, stride=1, padding=1)\n        self.encoder_conv2 = nn.Conv1d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1)\n        self.encoder_pool = nn.MaxPool1d(kernel_size=2, stride=2)\n        \n        self.flatten_size = None\n        \n        self.fc1 = nn.Linear(1, latent_dim)  # Placeholder, updated later\n        \n        # Decoder\n        self.fc2 = nn.Linear(latent_dim, 1)  # Placeholder, updated later\n        self.decoder_conv1 = nn.ConvTranspose1d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1)\n        self.decoder_conv2 = nn.ConvTranspose1d(in_channels=64, out_channels=input_dim, kernel_size=3, stride=1, padding=1)\n        self.decoder_upsample = nn.Upsample(scale_factor=2, mode='linear')\n        \n        self.relu = nn.ReLU()\n    \n    def forward(self, x):\n        batch_size = x.size(0)\n        x = x.permute(0, 2, 1)\n        \n        # Encoder\n        x = self.relu(self.encoder_conv1(x))\n        x = self.encoder_pool(x)\n        x = self.relu(self.encoder_conv2(x))\n        x = self.encoder_pool(x)\n        \n        # Dynamically calculate the flattened size\n        if self.flatten_size is None:\n            self.flatten_size = x.shape[1] * x.shape[2]\n            self.fc1 = nn.Linear(self.flatten_size, latent_dim)\n            self.fc2 = nn.Linear(latent_dim, self.flatten_size)\n        \n        # Latent space\n        x = x.view(batch_size, -1)  # Flatten\n        latent = self.relu(self.fc1(x))\n        \n        # Decoder\n        x = self.relu(self.fc2(latent))\n        x = x.view(batch_size, 32, -1)  # Reshape to match the decoder input shape\n        x = self.relu(self.decoder_conv1(x))\n        x = self.decoder_upsample(x)\n        x = self.decoder_conv2(x)\n        \n        return x.permute(0, 2, 1), latent  # Return to (batch_size, time_steps, features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:43.773942Z","iopub.execute_input":"2024-11-21T13:14:43.775084Z","iopub.status.idle":"2024-11-21T13:14:43.786957Z","shell.execute_reply.started":"2024-11-21T13:14:43.775037Z","shell.execute_reply":"2024-11-21T13:14:43.785337Z"}},"outputs":[],"execution_count":null},{"id":"faeecdb9-bfc9-4af0-a641-d21cc86f1c98","cell_type":"code","source":"input_dim = 100\nlatent_dim = 256  # Dimension of the bottleneck\nmodel = CNN1DEncoderDecoder(input_dim=input_dim, latent_dim=latent_dim)\nprint(model)\n\n# Example input: batch_size = 32, time_steps = 1280, features = 100\ninput_data = torch.randn(32, 1280, 100) #  x y z \noutput, latent = model(input_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T13:14:44.328558Z","iopub.execute_input":"2024-11-21T13:14:44.329064Z","iopub.status.idle":"2024-11-21T13:14:44.587430Z","shell.execute_reply.started":"2024-11-21T13:14:44.329026Z","shell.execute_reply":"2024-11-21T13:14:44.586167Z"}},"outputs":[],"execution_count":null},{"id":"81cfe38f-7dbe-49df-9c86-ffe584dc2f73","cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}