{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Children Internet Addiction Classification using PySpark and machine learning","metadata":{}},{"cell_type":"code","source":"# Install PySpark\n!pip install pyspark\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T13:42:42.348377Z","iopub.execute_input":"2024-10-06T13:42:42.348826Z","iopub.status.idle":"2024-10-06T13:43:33.845716Z","shell.execute_reply.started":"2024-10-06T13:42:42.348781Z","shell.execute_reply":"2024-10-06T13:43:33.844333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.1 Spark Setup and data load","metadata":{}},{"cell_type":"code","source":"from pyspark.sql import SparkSession\nfrom pyspark.sql.functions import col, when, mean, first, stddev, count\nfrom pyspark.sql.types import IntegerType, FloatType\n\n# Initialize Spark session\nspark = SparkSession.builder \\\n    .appName(\"HBN Severity Impairment Prediction\") \\\n    .getOrCreate()\n\n# Load train and test CSV files\ntrain_df = spark.read.csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv', header=True, inferSchema=True)\ntest_df = spark.read.csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv', header=True, inferSchema=True)\n\n# Load actigraphy (accelerometer) series data\nseries_train_df = spark.read.parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet')\nseries_test_df = spark.read.parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet')\n\n# Load data dictionary (optional, for reference)\ndata_dict = spark.read.csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv', header=True, inferSchema=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T14:14:22.568895Z","iopub.execute_input":"2024-10-06T14:14:22.569364Z","iopub.status.idle":"2024-10-06T14:14:28.911474Z","shell.execute_reply.started":"2024-10-06T14:14:22.569320Z","shell.execute_reply":"2024-10-06T14:14:28.910193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2024-10-06T14:11:55.537009Z","iopub.execute_input":"2024-10-06T14:11:55.537899Z","iopub.status.idle":"2024-10-06T14:11:55.548417Z","shell.execute_reply.started":"2024-10-06T14:11:55.537853Z","shell.execute_reply":"2024-10-06T14:11:55.547222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Preprocessing","metadata":{}},{"cell_type":"markdown","source":"# 2.1 Handling Missing Value","metadata":{}},{"cell_type":"code","source":"numeric_columns = [col for col, dtype in train_df.dtypes if dtype in ['int', 'double', 'float']]\n\n\nfor column in numeric_columns:\n    mean_value = train_df.agg(mean(col(column))).first()[0]\n    train_df = train_df.na.fill({column: mean_value})\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T14:14:33.749434Z","iopub.execute_input":"2024-10-06T14:14:33.750566Z","iopub.status.idle":"2024-10-06T14:14:49.909812Z","shell.execute_reply.started":"2024-10-06T14:14:33.750513Z","shell.execute_reply":"2024-10-06T14:14:49.908520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2 Feature Engineering on Time Series data ( Accelerotormeter data )","metadata":{}},{"cell_type":"code","source":"def extract_features(df):\n    return df.groupBy('id') \\\n        .agg(\n            mean('enmo').alias('mean_enmo'),\n            stddev('enmo').alias('stddev_enmo'),\n            mean('X').alias('mean_X'),\n            mean('Y').alias('mean_Y'),\n            mean('Z').alias('mean_Z'),\n            count('step').alias('total_steps')\n        )\n\n\nseries_train_features = extract_features(series_train_df)\nseries_test_features = extract_features(series_test_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T14:18:33.862554Z","iopub.execute_input":"2024-10-06T14:18:33.863413Z","iopub.status.idle":"2024-10-06T14:18:33.931267Z","shell.execute_reply.started":"2024-10-06T14:18:33.863367Z","shell.execute_reply":"2024-10-06T14:18:33.930252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.3 Merge Tabular and Time Series data","metadata":{}},{"cell_type":"code","source":"train_merged_df = train_df.join(series_train_features, on=\"id\", how=\"left\")\ntest_merged_df = test_df.join(series_test_features, on=\"id\", how=\"left\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T14:21:19.855061Z","iopub.execute_input":"2024-10-06T14:21:19.855578Z","iopub.status.idle":"2024-10-06T14:21:19.968930Z","shell.execute_reply.started":"2024-10-06T14:21:19.855532Z","shell.execute_reply":"2024-10-06T14:21:19.967689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}