{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:46.546973Z","iopub.execute_input":"2025-03-23T12:41:46.547366Z","iopub.status.idle":"2025-03-23T12:41:50.830028Z","shell.execute_reply.started":"2025-03-23T12:41:46.547332Z","shell.execute_reply":"2025-03-23T12:41:50.82859Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. dataset overview","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom matplotlib.ticker import PercentFormatter\n\n# Optional: define categorical dtype for seasons\nseason_dtype = pd.CategoricalDtype(categories=['Spring', 'Summer', 'Fall', 'Winter'])\n\n# Load data\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\n# Cast all columns ending with 'Season' to categorical\nseason_cols = [col for col in train_df.columns if col.endswith(\"Season\")]\n\ncommon_season_cols = [col for col in season_cols if col in test_df.columns]\n\ntrain_df[common_season_cols] = train_df[common_season_cols].apply(lambda col: col.astype(season_dtype))\ntest_df[common_season_cols] = test_df[common_season_cols].apply(lambda col: col.astype(season_dtype))\n\n\n# Keep only rows with known target\ntrain_clean = train_df[train_df['sii'].notna()].copy()\n\n# Calculate missing values and their ratio\nmissing_info = train_clean.isnull().sum().reset_index()\nmissing_info.columns = ['feature', 'missing_count']\nmissing_info['missing_ratio'] = missing_info['missing_count'] / len(train_clean)\nmissing_info = missing_info.sort_values(by='missing_ratio', ascending=False)\n\n# Plot missingness\nplt.figure(figsize=(7, 14))\nplt.title(f'Missing Data Overview (n={len(train_clean)})')\n\nplt.barh(\n    y=np.arange(len(missing_info)),\n    width=missing_info['missing_ratio'],\n    color='tomato',\n    label='Missing'\n)\n\nplt.barh(\n    y=np.arange(len(missing_info)),\n    width=1 - missing_info['missing_ratio'],\n    left=missing_info['missing_ratio'],\n    color='mediumseagreen',\n    label='Available'\n)\n\nplt.yticks(np.arange(len(missing_info)), missing_info['feature'])\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1))\nplt.xlabel(\"Percentage\")\nplt.xlim(0, 1)\nplt.legend()\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:42:47.407253Z","iopub.execute_input":"2025-03-23T12:42:47.407622Z","iopub.status.idle":"2025-03-23T12:42:49.021088Z","shell.execute_reply.started":"2025-03-23T12:42:47.407594Z","shell.execute_reply":"2025-03-23T12:42:49.019917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### for records with sii target value, missing value is in graph \n#### PCIAT-PCIAT_Total: the total score a participant received from the PCIAT","metadata":{}},{"cell_type":"markdown","source":"# 2. correlation","metadata":{}},{"cell_type":"code","source":"matching_nulls = (\n    ((train_df['sii'].isna()) == (train_df['PCIAT-PCIAT_Total'].isna()))\n).mean()\n\nprint(f\"% of rows where sii and PCIAT-PCIAT_Total both null or both present: {matching_nulls:.2%}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.388339Z","iopub.execute_input":"2025-03-23T12:41:52.388713Z","iopub.status.idle":"2025-03-23T12:41:52.396918Z","shell.execute_reply.started":"2025-03-23T12:41:52.388677Z","shell.execute_reply":"2025-03-23T12:41:52.395841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"valid_rows = train_df[train_df['sii'].notna() & train_df['PCIAT-PCIAT_Total'].notna()]\ncorrelation = valid_rows['sii'].corr(valid_rows['PCIAT-PCIAT_Total'])\nprint(f\"Correlation between sii and PCIAT-PCIAT_Total: {correlation:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.398259Z","iopub.execute_input":"2025-03-23T12:41:52.398608Z","iopub.status.idle":"2025-03-23T12:41:52.440702Z","shell.execute_reply.started":"2025-03-23T12:41:52.398569Z","shell.execute_reply":"2025-03-23T12:41:52.439594Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The correlation between sii (Severity Impairment Index) and PCIAT-PCIAT_Total is 0.8997, it is a strong positive linear relationship. This suggests that the sii class labels are highly influenced by or directly derived from the total PCIAT score. ","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.441747Z","iopub.execute_input":"2025-03-23T12:41:52.442029Z","iopub.status.idle":"2025-03-23T12:41:52.446394Z","shell.execute_reply.started":"2025-03-23T12:41:52.442004Z","shell.execute_reply":"2025-03-23T12:41:52.445104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure correct types\ntrain_df['sii'] = train_df['sii'].astype('Int64')  # nullable integer\ntrain_df = train_df[train_df['sii'].notna() & train_df['PCIAT-PCIAT_Total'].notna()].copy()\ntrain_df['sii'] = train_df['sii'].astype(int)\nsummary_stats = (\n    train_df.groupby('sii')['PCIAT-PCIAT_Total']\n    .agg(['count', 'mean', 'median', 'std'])\n    .sort_index()\n)\n\nprint(summary_stats)\nplt.figure(figsize=(8, 6))\nsns.boxplot(data=train_df, x='sii', y='PCIAT-PCIAT_Total', palette='Set2')\nplt.title(\"Distribution of PCIAT-Total across sii classes\")\nplt.xlabel(\"sii (severity class)\")\nplt.ylabel(\"PCIAT-PCIAT_Total\")\nplt.grid(True)\nplt.show()\n\nplt.figure(figsize=(8, 5))\nsns.countplot(data=train_df, x='sii', palette='viridis')\nplt.title('Class distribution of sii')\nplt.xlabel('sii')\nplt.ylabel('Count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.447249Z","iopub.execute_input":"2025-03-23T12:41:52.447499Z","iopub.status.idle":"2025-03-23T12:41:52.876208Z","shell.execute_reply.started":"2025-03-23T12:41:52.447476Z","shell.execute_reply":"2025-03-23T12:41:52.874977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count how many samples are in sii = 0\nsii_0_count = (train_df['sii'] == 0).sum()\n\n# Count total samples with non-null sii\ntotal_sii_count = train_df['sii'].notna().sum()\n\n# Print result and percentage\nprint(f\"sii = 0: {sii_0_count} out of {total_sii_count} ({sii_0_count / total_sii_count:.2%})\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.877534Z","iopub.execute_input":"2025-03-23T12:41:52.877887Z","iopub.status.idle":"2025-03-23T12:41:52.884478Z","shell.execute_reply.started":"2025-03-23T12:41:52.877848Z","shell.execute_reply":"2025-03-23T12:41:52.883166Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"half are in class 0, dataset is imbalanced","metadata":{}},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train_df.columns if f not in test_df.columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.887947Z","iopub.execute_input":"2025-03-23T12:41:52.888312Z","iopub.status.idle":"2025-03-23T12:41:52.909295Z","shell.execute_reply.started":"2025-03-23T12:41:52.888281Z","shell.execute_reply":"2025-03-23T12:41:52.907939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Enrollment Season\nenroll_counts = train_df['Basic_Demos-Enroll_Season'].value_counts()\n\nplt.figure(figsize=(5, 5))\nplt.pie(enroll_counts, labels=enroll_counts.index, autopct='%1.1f%%', startangle=140)\nplt.title('Enrollment Season Distribution')\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:52.910796Z","iopub.execute_input":"2025-03-23T12:41:52.911084Z","iopub.status.idle":"2025-03-23T12:41:53.069044Z","shell.execute_reply.started":"2025-03-23T12:41:52.91106Z","shell.execute_reply":"2025-03-23T12:41:53.067968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gender\n# boy=0 girl=0\ngender_counts = train_df['Basic_Demos-Sex'].value_counts().sort_index()\n\nplt.figure(figsize=(5, 5))\nplt.pie(gender_counts, labels=['Girls', 'Boys',], autopct='%1.1f%%', colors=['salmon','skyblue'], startangle=90)\nplt.title('Gender Distribution')\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:53.07019Z","iopub.execute_input":"2025-03-23T12:41:53.070532Z","iopub.status.idle":"2025-03-23T12:41:53.189171Z","shell.execute_reply.started":"2025-03-23T12:41:53.070492Z","shell.execute_reply":"2025-03-23T12:41:53.188116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Age Distribution by Gender\nfig, axes = plt.subplots(2, 1, figsize=(8, 6), sharex=True)\n\nfor i, gender in enumerate([0, 1]):\n    ax = axes[i]\n    age_counts = train_df[train_df['Basic_Demos-Sex'] == gender]['Basic_Demos-Age'].value_counts().sort_index()\n    ax.bar(age_counts.index, age_counts.values, color=[ 'coral','lightblue',][i])\n    ax.set_ylabel('Count')\n    ax.set_title(['Girls','Boys',][i])\n    ax.grid(True)\n\naxes[1].set_xlabel('Age (years)')\nplt.suptitle('Age Distribution by Gender')\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:53.19033Z","iopub.execute_input":"2025-03-23T12:41:53.190844Z","iopub.status.idle":"2025-03-23T12:41:53.709825Z","shell.execute_reply.started":"2025-03-23T12:41:53.190804Z","shell.execute_reply":"2025-03-23T12:41:53.70857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# target vs gender\nfig, axes = plt.subplots(2, 1, figsize=(8, 6), sharex=True, sharey=True)\n\nfor i, gender in enumerate([0, 1]):\n    ax = axes[i]\n    subset = train_df[train_df['Basic_Demos-Sex'] == gender]\n    sii_counts = subset['sii'].value_counts(normalize=True).sort_index()\n    ax.bar(sii_counts.index, sii_counts.values, color=['lightblue', 'coral'][i])\n    ax.set_title(['Boys', 'Girls'][i])\n    ax.set_ylabel('Proportion')\n    ax.set_xticks([0, 1, 2, 3])\n    ax.grid(True)\n\naxes[1].set_xlabel('sii Class (0 to 3)')\nplt.suptitle('sii Class Distribution by Gender')\nplt.gca().yaxis.set_major_formatter(PercentFormatter(xmax=1))\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:53.71118Z","iopub.execute_input":"2025-03-23T12:41:53.711569Z","iopub.status.idle":"2025-03-23T12:41:54.124331Z","shell.execute_reply.started":"2025-03-23T12:41:53.711528Z","shell.execute_reply":"2025-03-23T12:41:54.123051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n#  List of relevant numeric columns to analyze\nselected_features = [\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL',\n    'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n    'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n    'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n]\n\n# Filter the data to include only valid rows with complete PCIAT-Total\ncorr_data = train_df[train_df['PCIAT-PCIAT_Total'].notna()][selected_features]\n\n# Compute correlation matrix\ncorrelation_matrix = corr_data.corr()\n\n# Focus on correlation with PCIAT score\npciat_corr = correlation_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\n\n# Only keep strong correlations (above 0.1 or below -0.1)\nsignificant_corr = pciat_corr[abs(pciat_corr) > 0.1]\n\n# Show sorted bar chart of correlated features\nplt.figure(figsize=(8, 6))\nsignificant_corr.sort_values().plot(kind='barh', color='coral')\nplt.title('Top Correlations with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation')\nplt.ylabel('Feature')\nplt.grid(True, axis='x', linestyle='--', alpha=0.6)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:54.125557Z","iopub.execute_input":"2025-03-23T12:41:54.12595Z","iopub.status.idle":"2025-03-23T12:41:54.483252Z","shell.execute_reply.started":"2025-03-23T12:41:54.125913Z","shell.execute_reply":"2025-03-23T12:41:54.482151Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. time series - actigraphy ","metadata":{}},{"cell_type":"code","source":"actigraphy = pd.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:54.484406Z","iopub.execute_input":"2025-03-23T12:41:54.484829Z","iopub.status.idle":"2025-03-23T12:41:54.692288Z","shell.execute_reply.started":"2025-03-23T12:41:54.484787Z","shell.execute_reply":"2025-03-23T12:41:54.691228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load one parquet \nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\n\ndef visualize_actigraphy(subject_id, one_week_only=False, simple_view=False):\n    # Load data for one subject\n    file_path = f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={subject_id}/part-0.parquet'\n    df = pd.read_parquet(file_path)\n    \n    # Compute time in days (relative to PCIAT)\n    df['time_days'] = df['relative_date_PCIAT'] + df['time_of_day'] / 86400e9  # time_of_day is in nanoseconds\n\n    # Merge metadata for subject\n    meta = train_df[train_df['id'] == subject_id].iloc[0]\n    age = meta['Basic_Demos-Age']\n    sex = 'boy' if meta['Basic_Demos-Sex'] == 0 else 'girl'\n    \n    # Compute derived features\n    df['diff_seconds'] = df['time_days'].diff() * 86400\n    df['norm'] = np.sqrt(df['X']**2 + df['Y']**2 + df['Z']**2)\n\n    # Filter time window (first week) and exclude non-wear\n    if one_week_only:\n        day0 = np.ceil(df['time_days'].min())\n        mask = (df['time_days'] >= day0) & (df['time_days'] <= day0 + 21)\n        mask &= (df['non-wear_flag'] == 0)\n    else:\n        mask = np.full(len(df), True)\n\n    # Feature selection\n    if simple_view:\n        features = [('enmo', 'forestgreen'), ('light', 'orange')]\n    else:\n        features = [\n            ('X', 'm'), ('Y', 'm'), ('Z', 'm'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n        ]\n\n    # Plot\n    fig, axes = plt.subplots(len(features), 1, figsize=(12, 1.2 * len(features)), sharex=True)\n\n    for ax, (col, color) in zip(axes, features):\n        ax.set_facecolor('#f5f5f5')\n        ax.scatter(df['time_days'][mask], df[col][mask], color=color, s=1, label=col)\n        ax.legend(loc='upper left', frameon=True, facecolor='#f5f5f5')\n        if col == 'diff_seconds':\n            ax.set_ylim(-1, 25)\n        ax.set_ylabel(col)\n\n    axes[-1].set_xlabel('Days since PCIAT')\n    axes[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    axes[0].set_title(f'Subject {subject_id} – {sex}, Age {age}')\n    plt.tight_layout()\n    plt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:54.693372Z","iopub.execute_input":"2025-03-23T12:41:54.693692Z","iopub.status.idle":"2025-03-23T12:41:54.706191Z","shell.execute_reply.started":"2025-03-23T12:41:54.693664Z","shell.execute_reply":"2025-03-23T12:41:54.704871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_actigraphy('00115b9f', one_week_only=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:54.707282Z","iopub.execute_input":"2025-03-23T12:41:54.707598Z","iopub.status.idle":"2025-03-23T12:41:56.286333Z","shell.execute_reply.started":"2025-03-23T12:41:54.707573Z","shell.execute_reply":"2025-03-23T12:41:56.285247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_actigraphy('00f332d1', one_week_only=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:41:56.287861Z","iopub.execute_input":"2025-03-23T12:41:56.28816Z","iopub.status.idle":"2025-03-23T12:41:59.004191Z","shell.execute_reply.started":"2025-03-23T12:41:56.288115Z","shell.execute_reply":"2025-03-23T12:41:59.003079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. feature engineering","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm import tqdm\nfrom concurrent.futures import ThreadPoolExecutor\n\ndef extract_summary_stats(folder_name, base_path):\n    \"\"\"Loads a .parquet sensor file and returns flattened statistical summary.\"\"\"\n    full_path = os.path.join(base_path, folder_name, 'part-0.parquet')\n    df = pd.read_parquet(full_path)\n    \n    # Remove unwanted column if present\n    df = df.drop(columns='step', errors='ignore')\n\n    # Compute descriptive stats and flatten\n    stats_array = df.describe().values.ravel()\n    subject_code = folder_name.split('=')[-1]\n    \n    return stats_array, subject_code\n\ndef build_feature_matrix(parquet_root):\n    \"\"\"Processes all sensor folders and builds a dataframe of statistical features.\"\"\"\n    subject_dirs = os.listdir(parquet_root)\n    \n    with ThreadPoolExecutor() as executor:\n        extracted = list(tqdm(executor.map(lambda d: extract_summary_stats(d, parquet_root), subject_dirs), total=len(subject_dirs)))\n\n    feature_data, subject_ids = zip(*extracted)\n    df_out = pd.DataFrame(feature_data, columns=[f\"ts_feat_{i}\" for i in range(len(feature_data[0]))])\n    df_out['id'] = subject_ids\n    return df_out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:43:23.714252Z","iopub.execute_input":"2025-03-23T12:43:23.714729Z","iopub.status.idle":"2025-03-23T12:43:23.725573Z","shell.execute_reply.started":"2025-03-23T12:43:23.714689Z","shell.execute_reply":"2025-03-23T12:43:23.724198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.preprocessing import StandardScaler\n\nclass FeatureCompressor(nn.Module):\n    def __init__(self, input_size, latent_size):\n        super().__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_size, latent_size * 4),\n            nn.ReLU(),\n            nn.Linear(latent_size * 4, latent_size * 2),\n            nn.ReLU(),\n            nn.Linear(latent_size * 2, latent_size)\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(latent_size, input_size * 2),\n            nn.ReLU(),\n            nn.Linear(input_size * 2, input_size),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        return self.decoder(self.encoder(x))\n\ndef autoencode_features(df, latent_size=40, max_epochs=30, batch=64):\n    \"\"\"Encodes high-dimensional features using an unsupervised autoencoder.\"\"\"\n    scaler = StandardScaler()\n    normalized = scaler.fit_transform(df)\n\n    tensor_data = torch.FloatTensor(normalized)\n    model = FeatureCompressor(tensor_data.shape[1], latent_size)\n    optimizer = optim.Adam(model.parameters(), lr=1e-3)\n    loss_fn = nn.MSELoss()\n\n    for epoch in range(max_epochs):\n        for start in range(0, len(tensor_data), batch):\n            batch_input = tensor_data[start:start+batch]\n            optimizer.zero_grad()\n            output = model(batch_input)\n            loss = loss_fn(output, batch_input)\n            loss.backward()\n            optimizer.step()\n        if (epoch + 1) % 5 == 0:\n            print(f\"[Epoch {epoch+1}/{max_epochs}] loss = {loss.item():.4f}\")\n\n    with torch.no_grad():\n        compressed = model.encoder(tensor_data).numpy()\n\n    return pd.DataFrame(compressed, columns=[f'compressed_{i+1}' for i in range(latent_size)])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:43:36.327923Z","iopub.execute_input":"2025-03-23T12:43:36.32833Z","iopub.status.idle":"2025-03-23T12:43:36.339987Z","shell.execute_reply.started":"2025-03-23T12:43:36.328296Z","shell.execute_reply":"2025-03-23T12:43:36.33858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def engineer_features(df):\n    \"\"\"Creates domain-specific features for tabular input.\"\"\"\n    # Drop season-related columns\n    season_related = [col for col in df.columns if 'Season' in col]\n    df = df.drop(columns=season_related)\n\n    # Add new features through arithmetic combinations\n    df['BMIxAge'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['ScreenTimexAge'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMIxScreenTime'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    \n    df['FatToBMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMIvsFat'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMIvsFat'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    \n    df['LST_ratio'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['FatxBMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['FatxDEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    \n    df['BMR_per_kg'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_per_kg'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['Muscle_per_height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    \n    df['MuscleFatRatio'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['WaterRatio'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_percent'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:43:46.015502Z","iopub.execute_input":"2025-03-23T12:43:46.016017Z","iopub.status.idle":"2025-03-23T12:43:46.024259Z","shell.execute_reply.started":"2025-03-23T12:43:46.015971Z","shell.execute_reply":"2025-03-23T12:43:46.022848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.impute import KNNImputer\n\n# Load core data files\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Load time-series statistical summaries\ntrain_series_raw = build_feature_matrix(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_series_raw = build_feature_matrix(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n#  Apply autoencoder-based compression\ncompressed_train_series = autoencode_features(train_series_raw.drop(columns='id'), latent_size=60, max_epochs=100, batch=32)\ncompressed_test_series = autoencode_features(test_series_raw.drop(columns='id'), latent_size=60, max_epochs=100, batch=32)\n\n# Merge compressed features into main data\ncompressed_train_series['id'] = train_series_raw['id']\ncompressed_test_series['id'] = test_series_raw['id']\n\ntrain_df = pd.merge(train_df, compressed_train_series, on='id', how='left')\ntest_df = pd.merge(test_df, compressed_test_series, on='id', how='left')\n\n# Handle missing values using KNN-based imputation\nimputer = KNNImputer(n_neighbors=5)\nnumerics_only = train_df.select_dtypes(include=['int64', 'float64']).columns\ntrain_imputed_values = imputer.fit_transform(train_df[numerics_only])\ntrain_filled = pd.DataFrame(train_imputed_values, columns=numerics_only)\n\n# Convert back sii to integer after imputation\ntrain_filled['sii'] = train_filled['sii'].round().astype(int)\n\n# Add back non-numeric columns\nfor col in train_df.columns:\n    if col not in numerics_only:\n        train_filled[col] = train_df[col]\n\n# Replace original training set with filled version\ntrain_df = train_filled.copy()\n\n#  Add derived features (your rewritten function from earlier)\ntrain_df = engineer_features(train_df)\ntest_df = engineer_features(test_df)\n\n#  Drop low-info rows (too many nulls), remove ID\ntrain_df = train_df.dropna(thresh=10, axis=0).drop(columns='id', errors='ignore')\ntest_df = test_df.drop(columns='id', errors='ignore')\n\n#  Define feature columns (original features + derived + autoencoded)\ncore_features = [\n    'Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score',\n    'Physical-BMI', 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n    'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone',\n    'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n    'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n    'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI',\n    'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n    'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total',\n    'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', 'PreInt_EduHx-computerinternet_hoursday', 'sii'\n]\n\n# Newly engineered features\nderived_cols = [\n    'BMIxAge', 'ScreenTimexAge', 'BMIxScreenTime', 'FatToBMI',\n    'FFMIvsFat', 'FMIvsFat', 'LST_ratio', 'FatxDEE',\n    'BMR_per_kg', 'DEE_per_kg',\n    'Muscle_per_height', 'MuscleFatRatio', 'WaterRatio', 'ICW_percent'\n]\n\n\n\n# Add autoencoded column names\nautoencoded_cols = compressed_train_series.columns.drop('id').tolist()\n\n# Combine all feature columns\nall_features = core_features[:-1] + derived_cols + autoencoded_cols  # exclude 'sii' for now\n\n# Final training data with target\ntrain_df = train_df[all_features + ['sii']]\ntest_df = test_df[all_features]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:44:04.382385Z","iopub.execute_input":"2025-03-23T12:44:04.382768Z","iopub.status.idle":"2025-03-23T12:45:42.606171Z","shell.execute_reply.started":"2025-03-23T12:44:04.382726Z","shell.execute_reply":"2025-03-23T12:45:42.605021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\nnumeric_cols = train.select_dtypes(include=['number']).columns\ninf_mask = np.isinf(train[numeric_cols].to_numpy())\n\nif inf_mask.any():\n    train_df[numeric_cols] = train_df[numeric_cols].replace([np.inf, -np.inf], np.nan)\n\ndef compute_qwk(true, predicted):\n    \"\"\"Compute the quadratic weighted kappa score.\"\"\"\n    return cohen_kappa_score(true, predicted, weights='quadratic')\n\ndef round_by_thresholds(preds, cuts):\n    \"\"\"Map continuous outputs to class labels based on thresholds.\"\"\"\n    return np.digitize(preds, bins=cuts)\n\ndef evaluate_qwk_loss(thresholds, y_true, preds_raw):\n    \"\"\"Evaluate predictions using QWK as a loss (negative for optimization).\"\"\"\n    predicted_classes = round_by_thresholds(preds_raw, thresholds)\n    return -compute_qwk(y_true, predicted_classes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:53:08.157373Z","iopub.execute_input":"2025-03-23T12:53:08.157792Z","iopub.status.idle":"2025-03-23T12:53:08.169718Z","shell.execute_reply.started":"2025-03-23T12:53:08.157762Z","shell.execute_reply":"2025-03-23T12:53:08.168628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(X, y, model_type=\"lgbm\", thresholds=[0.5, 1.5, 2.5], seed=42):\n    from sklearn.model_selection import train_test_split\n\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)\n\n    # Clean inf and fill NaNs\n    for df in [X_train, X_val]:\n        num_cols = df.select_dtypes(include=['number']).columns\n        df[num_cols] = df[num_cols].replace([np.inf, -np.inf], np.nan).fillna(0)\n\n    if model_type == \"xgb\":\n        model = xgb.XGBRegressor(\n            learning_rate=0.05,\n            max_depth=6,\n            n_estimators=200,\n            subsample=0.8,\n            colsample_bytree=0.8,\n            reg_alpha=1,\n            reg_lambda=5,\n            tree_method='hist',\n            random_state=seed,\n        )\n    elif model_type == \"lgbm\":\n        model = lgb.LGBMRegressor(\n            learning_rate=0.046,\n            max_depth=12,\n            num_leaves=478,\n            min_data_in_leaf=13,\n            feature_fraction=0.893,\n            bagging_fraction=0.784,\n            bagging_freq=4,\n            reg_alpha=10,\n            reg_lambda=0.01,\n            n_jobs=-1,\n            random_state=seed,\n        )\n    elif model_type == \"catboost\":\n        model = CatBoostRegressor(\n            learning_rate=0.05,\n            depth=6,\n            iterations=200,\n            l2_leaf_reg=10,\n            task_type='CPU',  #  'GPU' \n            verbose=0,\n            random_seed=seed,\n        )\n    else:\n        raise ValueError(\"Model type not supported.\")\n\n    model.fit(X_train, y_train)\n    raw_preds = model.predict(X_val)\n    preds = convert_to_class(raw_preds, thresholds)\n    qwk = qwk_score(y_val, preds)\n    print(f\"{model_type.upper()} QWK Score: {qwk:.4f}\")\n\n    return model, qwk, X_val, y_val\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:53:15.832014Z","iopub.execute_input":"2025-03-23T12:53:15.832389Z","iopub.status.idle":"2025-03-23T12:53:15.841826Z","shell.execute_reply.started":"2025-03-23T12:53:15.832361Z","shell.execute_reply":"2025-03-23T12:53:15.840564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_model, lgbm_qwk, X_val, y_val = train_model(X_data, y_data, model_type=\"lgbm\")\nxgb_model, xgb_qwk, _, _ = train_model(X_data, y_data, model_type=\"xgb\")\ncat_model, cat_qwk, _, _ = train_model(X_data, y_data, model_type=\"catboost\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:53:20.246651Z","iopub.execute_input":"2025-03-23T12:53:20.247035Z","iopub.status.idle":"2025-03-23T12:53:26.669535Z","shell.execute_reply.started":"2025-03-23T12:53:20.247006Z","shell.execute_reply":"2025-03-23T12:53:26.66807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds_lgb = lgbm_model.predict(test_df)\ntest_preds_xgb = xgb_model.predict(test_df)\ntest_preds_cat = cat_model.predict(test_df)\n\n# Weighted voting ensemble\ntest_ensemble_preds = (\n    0.8 * test_preds_lgb + \n    0.1 * test_preds_xgb + \n    0.1 * test_preds_cat\n)\n\n# Convert raw predictions to class labels using your thresholds\ndef convert_to_class(preds, cuts=[0.5, 1.5, 2.5]):\n    return np.digitize(preds, bins=cuts)\n\nfinal_test_classes = convert_to_class(test_ensemble_preds)\n\ndef export_submission(preds, ids, filename=\"submission.csv\"):\n    \"\"\"\n    Save predictions into a Kaggle submission CSV.\n    \"\"\"\n    df_submit = pd.DataFrame({\n        \"id\": ids,\n        \"sii\": preds\n    })\n    df_submit.to_csv(filename, index=False)\n    print(f\"Submission saved as: {filename}\")\n    return df_submit\n\n\nsubmission = export_submission(final_test_classes, test['id'])\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-23T12:53:27.987374Z","iopub.execute_input":"2025-03-23T12:53:27.987729Z","iopub.status.idle":"2025-03-23T12:53:28.026794Z","shell.execute_reply.started":"2025-03-23T12:53:27.987701Z","shell.execute_reply":"2025-03-23T12:53:28.02561Z"}},"outputs":[],"execution_count":null}]}