{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  Learning\n\nNotes about what I've learned during this challenge so far:\n1. polars vs. pandas: polars has faster processing time\n2. cohen's kappa: a score that tells you how much models agree with each other while considering the chance of random agreement. -1 to 1, where 1 means perfect agreement, 0 means agreement is no better than random chance.","metadata":{"_kg_hide-input":true}},{"cell_type":"markdown","source":"# Load libraries and data","metadata":{}},{"cell_type":"code","source":"#less mature and limited ecosystem than pandas, but faster for large datasets\nimport polars as pl\n#column selector\nimport polars.selectors as cs\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n#classes for configuring tick locating and formatting\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport lightgbm\n#color for text\nfrom colorama import Fore, Style\n#function minimization\nfrom scipy.optimize import minimize\n\n#for unbalanced datasets, makes sure each fold is representative\n#https://medium.com/@juanc.olamendy/a-comprehensive-guide-to-stratified-k-fold-cross-validation-for-unbalanced-data-014691060f17\nfrom sklearn.model_selection import StratifiedKFold\n\n#inter-annotator agreement\n#https://surge-ai.medium.com/inter-annotator-agreement-an-introduction-to-cohens-kappa-statistic-dcc15ffa5ac4\nfrom sklearn.metrics import cohen_kappa_score, ConfusionMatrixDisplay","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:39:19.586233Z","iopub.execute_input":"2024-12-11T13:39:19.586731Z","iopub.status.idle":"2024-12-11T13:39:22.698291Z","shell.execute_reply.started":"2024-12-11T13:39:19.586685Z","shell.execute_reply":"2024-12-11T13:39:22.696838Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#define target classes\ntarget_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:39:27.305075Z","iopub.execute_input":"2024-12-11T13:39:27.305871Z","iopub.status.idle":"2024-12-11T13:39:27.312089Z","shell.execute_reply.started":"2024-12-11T13:39:27.305804Z","shell.execute_reply":"2024-12-11T13:39:27.310675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#create enumerated data type using polars library\nseason_dtype = pl.Enum(['Winter', 'Spring', 'Summer', 'Fall'])\n\n#select columns ending with Season and cast to the season_dtype data type\n#ensures columns containing season data are standardized to an enumerated type to improve data consistency\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:40:32.793876Z","iopub.execute_input":"2024-12-11T13:40:32.794296Z","iopub.status.idle":"2024-12-11T13:40:32.818160Z","shell.execute_reply.started":"2024-12-11T13:40:32.794254Z","shell.execute_reply":"2024-12-11T13:40:32.816307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory data analysis","metadata":{}},{"cell_type":"markdown","source":"## View data","metadata":{}},{"cell_type":"code","source":"train #(3960, 82)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:40:47.043417Z","iopub.execute_input":"2024-12-11T13:40:47.043847Z","iopub.status.idle":"2024-12-11T13:40:47.060284Z","shell.execute_reply.started":"2024-12-11T13:40:47.043806Z","shell.execute_reply":"2024-12-11T13:40:47.058930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test #(20, 59)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:40:53.402853Z","iopub.execute_input":"2024-12-11T13:40:53.403275Z","iopub.status.idle":"2024-12-11T13:40:53.418923Z","shell.execute_reply.started":"2024-12-11T13:40:53.403233Z","shell.execute_reply":"2024-12-11T13:40:53.417400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.schema","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:40:57.736043Z","iopub.execute_input":"2024-12-11T13:40:57.736487Z","iopub.status.idle":"2024-12-11T13:40:57.752435Z","shell.execute_reply.started":"2024-12-11T13:40:57.736444Z","shell.execute_reply":"2024-12-11T13:40:57.751104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.schema","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:41:03.987474Z","iopub.execute_input":"2024-12-11T13:41:03.987932Z","iopub.status.idle":"2024-12-11T13:41:03.998917Z","shell.execute_reply.started":"2024-12-11T13:41:03.987889Z","shell.execute_reply":"2024-12-11T13:41:03.997669Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Missingness","metadata":{}},{"cell_type":"code","source":"#sum of nulls per feature\ntrain.select(pl.col(\"*\").is_null().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:41:10.343420Z","iopub.execute_input":"2024-12-11T13:41:10.343807Z","iopub.status.idle":"2024-12-11T13:41:10.364463Z","shell.execute_reply.started":"2024-12-11T13:41:10.343761Z","shell.execute_reply":"2024-12-11T13:41:10.363240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.select(pl.col(\"*\").is_null().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T13:41:16.223469Z","iopub.execute_input":"2024-12-11T13:41:16.223948Z","iopub.status.idle":"2024-12-11T13:41:16.234497Z","shell.execute_reply.started":"2024-12-11T13:41:16.223905Z","shell.execute_reply":"2024-12-11T13:41:16.232927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check missingness for all features\nmissing_count = (\n    train\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(train)).alias('null_ratio'))\n)\nplt.figure(figsize=(5, 15))\nplt.title('Missing values for training dataset')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='navy', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='orange', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:33:37.793759Z","iopub.execute_input":"2024-12-11T14:33:37.794169Z","iopub.status.idle":"2024-12-11T14:33:38.910740Z","shell.execute_reply.started":"2024-12-11T14:33:37.794132Z","shell.execute_reply":"2024-12-11T14:33:38.909489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check missingness only when sii is not null\nsupervised_usable = (\n    train\n    .filter(pl.col('sii').is_not_null())\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\nplt.figure(figsize=(6, 15))\nplt.title(f'Missing values for the {len(supervised_usable)} observations with the target (sii)')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='navy', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='orange', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:33:24.129303Z","iopub.execute_input":"2024-12-11T14:33:24.129705Z","iopub.status.idle":"2024-12-11T14:33:25.244754Z","shell.execute_reply.started":"2024-12-11T14:33:24.129669Z","shell.execute_reply":"2024-12-11T14:33:25.243436Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Frequencies","metadata":{}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\n\nseasons = vc.get_column('Basic_Demos-Enroll_Season')\ncounts = vc.get_column('count')\n\nsort = sorted(range(len(counts)), key = lambda i: counts[i], reverse = True)\nseasons = [seasons[i] for i in sort]\ncounts = [counts[i] for i in sort]\n\nplt.bar(seasons, counts, color = \"orange\")\nplt.xlabel(\"Seasons\")\nplt.ylabel(\"Counts\")\nplt.title(\"Enrollment season\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:54.516545Z","iopub.execute_input":"2024-12-11T14:32:54.516935Z","iopub.status.idle":"2024-12-11T14:32:54.757914Z","shell.execute_reply.started":"2024-12-11T14:32:54.516894Z","shell.execute_reply":"2024-12-11T14:32:54.756564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n    ax.bar(vc.get_column('sii'),\n           vc.get_column('count') / vc.get_column('count').sum(),\n           color=['orange', 'navy'][sex],\n           label=['Male', 'Female'][sex])\n    ax.set_xticks(np.arange(4), target_labels)\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n    ax.set_ylabel('Count')\n    ax.legend()\nplt.suptitle('Severity impairment index by sex')\naxs.ravel()[1].set_xlabel('Severity impairment index (sii)')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:36:58.488104Z","iopub.execute_input":"2024-12-11T14:36:58.488572Z","iopub.status.idle":"2024-12-11T14:36:58.833166Z","shell.execute_reply.started":"2024-12-11T14:36:58.488532Z","shell.execute_reply":"2024-12-11T14:36:58.831734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n           vc.get_column('count'),\n           color=['orange', 'navy'][sex],\n           label=['Male', 'Female'][sex])\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Age distribution')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:31.163378Z","iopub.execute_input":"2024-12-11T14:32:31.163790Z","iopub.status.idle":"2024-12-11T14:32:31.549071Z","shell.execute_reply.started":"2024-12-11T14:32:31.163751Z","shell.execute_reply":"2024-12-11T14:32:31.547688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The target (sii) is available for participants for whom we have results of the Parent-Child Internet Addiction Test (PCIAT), and it is a function of the PCIAT total score: \n\n* sii = 0 is defined as PCIAT 0-30\n* sii = 1 is defined as PCIAT 31-49\n* sii = 2 is defined as PCIAT 50-79\n* sii = 3 is defined as PCIAT 80+","metadata":{}},{"cell_type":"code","source":"(train\n .select(pl.col('PCIAT-PCIAT_Total'))\n .group_by(train.get_column('sii'))\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n .sort('sii')\n)\n#the dataset is imbalanced. almost all samples are in class 0, very few in class 3.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:07:43.992856Z","iopub.execute_input":"2024-12-11T14:07:43.993278Z","iopub.status.idle":"2024-12-11T14:07:44.017593Z","shell.execute_reply.started":"2024-12-11T14:07:43.993243Z","shell.execute_reply":"2024-12-11T14:07:44.016405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Selected other features:","metadata":{}},{"cell_type":"markdown","source":"### Body mass index\n\nCompare BMI measurements to assess data quality.","metadata":{}},{"cell_type":"code","source":"#checking BMI difference for those who had it measured 2x\nbmi_ratio = train.select(pl.col('Physical-BMI') / pl.col('BIA-BIA_BMI')).to_series()\ncolor = (bmi_ratio < 0.7) | (bmi_ratio > 1.3) # red if difference > 30 %\n\nplt.scatter(train.get_column('Physical-BMI'),\n            train.get_column('BIA-BIA_BMI'),\n            s=6,\n            cmap='coolwarm',\n            c=color)\nplt.gca().set_aspect('equal')\nplt.xlabel('Physical-BMI')\nplt.ylabel('BIA-BIA_BMI')\nplt.title('How much can BMI change in a year?')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:34:10.085033Z","iopub.execute_input":"2024-12-11T14:34:10.085497Z","iopub.status.idle":"2024-12-11T14:34:10.376284Z","shell.execute_reply.started":"2024-12-11T14:34:10.085453Z","shell.execute_reply":"2024-12-11T14:34:10.374982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Blood pressure","metadata":{}},{"cell_type":"code","source":"color = train.get_column('Physical-Systolic_BP') <= train.get_column('Physical-Diastolic_BP')\n\nplt.scatter(train.get_column('Physical-Diastolic_BP'),\n            train.get_column('Physical-Systolic_BP'),\n            s=6,\n            cmap='coolwarm',\n            c=color)\nplt.gca().set_aspect('equal')\nplt.plot([0, 200], [0, 200], color='gray')\nplt.xlabel('Diastolic blood pressure')\nplt.ylabel('Systolic blood pressure')\nplt.title('Systolic vs. diastolic blood pressure')\nplt.xticks(np.linspace(0, 200, 5))\nplt.yticks(np.linspace(0, 200, 5))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:35:34.111605Z","iopub.execute_input":"2024-12-11T14:35:34.112036Z","iopub.status.idle":"2024-12-11T14:35:34.393836Z","shell.execute_reply.started":"2024-12-11T14:35:34.111997Z","shell.execute_reply":"2024-12-11T14:35:34.392056Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Heart rate","metadata":{}},{"cell_type":"code","source":"#this is meant to check for outliers, but it's only cutting off at 50 on the left tail\n#would it be better to use IQR to check for outliers instead?\nvc = train.get_column('Physical-HeartRate').value_counts()\ncolor = np.where(vc.get_column('Physical-HeartRate') < 50, 'r', 'navy')\nplt.figure(figsize=(10, 3))\nplt.title('Heart rate with outliers')\nplt.bar(vc.get_column('Physical-HeartRate'), vc.get_column('count'), color=color)\nplt.xlabel('Physical-HeartRate')\nplt.ylabel('count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:45:59.938136Z","iopub.execute_input":"2024-12-11T14:45:59.938555Z","iopub.status.idle":"2024-12-11T14:46:00.314708Z","shell.execute_reply.started":"2024-12-11T14:45:59.938509Z","shell.execute_reply":"2024-12-11T14:46:00.313446Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sleep disturbance","metadata":{}},{"cell_type":"code","source":"vc = train.get_column('SDS-SDS_Total_Raw').value_counts()\nplt.figure(figsize=(6, 2))\nplt.title('Sleep disturbance scale')\nplt.bar(vc.get_column('SDS-SDS_Total_Raw'), vc.get_column('count'), color='navy')\nplt.xlabel('SDS-SDS_Total_Raw')\nplt.ylabel('count')\nplt.show()\n\nplt.title('Sleep disturbance: conversion from raw to T')\nplt.scatter(train.get_column('SDS-SDS_Total_Raw'),\n            train.get_column('SDS-SDS_Total_T'),\n            color='navy')\nplt.xlabel('SDS-SDS_Total_Raw')\nplt.ylabel('SDS-SDS_Total_T')\nplt.show()\n\nvc = train.get_column('SDS-SDS_Total_T').value_counts()\nplt.figure(figsize=(6, 2))\nplt.title('Sleep disturbance scale')\nplt.bar(vc.get_column('SDS-SDS_Total_T'), vc.get_column('count'), color='navy')\nplt.xlabel('SDS-SDS_Total_T')\nplt.ylabel('count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:46:32.739432Z","iopub.execute_input":"2024-12-11T14:46:32.739874Z","iopub.status.idle":"2024-12-11T14:46:33.622464Z","shell.execute_reply.started":"2024-12-11T14:46:32.739833Z","shell.execute_reply":"2024-12-11T14:46:33.621305Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are many outliers across various fields. We have to decide whether to keep them, modify them (winsorizing) or drop them.\n","metadata":{}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=ffed1dd5/part-0.parquet')\nactigraphy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T21:50:24.009294Z","iopub.execute_input":"2024-11-18T21:50:24.009807Z","iopub.status.idle":"2024-11-18T21:50:24.150176Z","shell.execute_reply.started":"2024-11-18T21:50:24.009768Z","shell.execute_reply":"2024-11-18T21:50:24.148959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pytorch-tabnet\n\n#Learning problems can be roughly categorized as supervised or unsupervised. \n#Supervised learning: goal is to predict the value of an outcome measure based on a number of input measures;\n    #“supervised” because of the presence of the outcome variable to guide the learning process\n#Unsupervised learning: no outcome measure, goal is to describe the associations and patterns among a set of input measures.","metadata":{"execution":{"iopub.status.busy":"2024-11-08T18:46:36.187687Z","iopub.execute_input":"2024-11-08T18:46:36.188263Z","iopub.status.idle":"2024-11-08T18:46:52.157127Z","shell.execute_reply.started":"2024-11-08T18:46:36.188200Z","shell.execute_reply":"2024-11-08T18:46:52.155373Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#other libraries loaded in some notebooks i've encountered, tossing here to look at & understand what they are, what they do\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nimport torch\n\nimport os\nimport re\nfrom sklearn.base import clone\n\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-08T18:52:51.388661Z","iopub.execute_input":"2024-11-08T18:52:51.389154Z","iopub.status.idle":"2024-11-08T18:53:14.364610Z","shell.execute_reply.started":"2024-11-08T18:52:51.389105Z","shell.execute_reply":"2024-11-08T18:53:14.363050Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Correlation","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\ncorr_matrix = supervised_usable.select([\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n    # Add other relevant columns\n]).to_pandas().corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\nprint(filtered_corr)\n\nplt.figure(figsize=(8, 6))\nfiltered_corr.sort_values().plot(kind='barh', color='navy')\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-11T14:41:14.464065Z","iopub.execute_input":"2024-12-11T14:41:14.464508Z","iopub.status.idle":"2024-12-11T14:41:14.822452Z","shell.execute_reply.started":"2024-12-11T14:41:14.464469Z","shell.execute_reply":"2024-12-11T14:41:14.821179Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"execution":{"iopub.status.busy":"2024-12-11T14:38:14.828645Z","iopub.execute_input":"2024-12-11T14:38:14.829079Z","iopub.status.idle":"2024-12-11T14:38:14.978826Z","shell.execute_reply.started":"2024-12-11T14:38:14.829039Z","shell.execute_reply":"2024-12-11T14:38:14.977621Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n    sample = train.filter(pl.col('id') == id)\n    age = sample.get_column('Basic_Demos-Age').item()\n    sex = ['Male', 'Female'][sample.get_column('Basic_Demos-Sex').item()]\n    actigraphy = (\n        actigraphy\n        .with_columns(\n            (day.diff() * 86400).alias('diff_seconds'),\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n        )\n    )\n\n    if only_one_week:\n        start = np.ceil(day.min())\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n    else:\n        mask = np.full(len(day), True)\n        \n    if small:\n        timelines = [\n            ('enmo', 'forestgreen'),\n            ('light', 'orange'),\n        ]\n    else:\n        timelines = [\n            ('X', 'm'),\n            ('Y', 'm'),\n            ('Z', 'm'),\n#             ('norm', 'c'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n    #         ('diff_seconds', 'k'),\n        ]\n        \n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n    for ax, (feature, color) in zip(axs, timelines):\n        ax.set_facecolor('#eeeeee')\n        ax.scatter(day.to_numpy()[mask],\n                   actigraphy.get_column(feature).to_numpy()[mask],\n                   color=color, label=feature, s=1)\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n        if feature == 'diff_seconds':\n            ax.set_ylim(-0.5, 20.5)\n    axs[-1].set_xlabel('day')\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.tight_layout()\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n    plt.show()\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T14:38:20.021568Z","iopub.execute_input":"2024-12-11T14:38:20.021984Z","iopub.status.idle":"2024-12-11T14:38:22.573346Z","shell.execute_reply.started":"2024-12-11T14:38:20.021944Z","shell.execute_reply":"2024-12-11T14:38:22.572002Z"},"trusted":true},"outputs":[],"execution_count":null}]}