{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nimport numpy as np\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-26T18:08:44.560442Z","iopub.execute_input":"2024-09-26T18:08:44.560838Z","iopub.status.idle":"2024-09-26T18:08:54.281906Z","shell.execute_reply.started":"2024-09-26T18:08:44.560800Z","shell.execute_reply":"2024-09-26T18:08:54.280854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"physical_devices = tf.config.list_physical_devices('GPU')\nif len(physical_devices) > 0:\n    tf.config.experimental.set_memory_growth(physical_devices[0], True)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:08:58.219927Z","iopub.execute_input":"2024-09-26T18:08:58.221097Z","iopub.status.idle":"2024-09-26T18:08:58.443791Z","shell.execute_reply.started":"2024-09-26T18:08:58.221046Z","shell.execute_reply":"2024-09-26T18:08:58.442623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_folder = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet'\ntrain_csv_path = '/kaggle/input/child-mind-institute-problematic-internet-use/train.csv'\ntest_csv_path='/kaggle/input/child-mind-institute-problematic-internet-use/test.csv'","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:00.818045Z","iopub.execute_input":"2024-09-26T18:09:00.818837Z","iopub.status.idle":"2024-09-26T18:09:00.823201Z","shell.execute_reply.started":"2024-09-26T18:09:00.818796Z","shell.execute_reply":"2024-09-26T18:09:00.822128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df= pd.read_csv(test_csv_path)\ncolumns= test_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:03.450796Z","iopub.execute_input":"2024-09-26T18:09:03.451194Z","iopub.status.idle":"2024-09-26T18:09:03.471541Z","shell.execute_reply.started":"2024-09-26T18:09:03.451155Z","shell.execute_reply":"2024-09-26T18:09:03.470800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df= pd.read_csv(train_csv_path)\ntrain_columns= train_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:05.466196Z","iopub.execute_input":"2024-09-26T18:09:05.467143Z","iopub.status.idle":"2024-09-26T18:09:05.526465Z","shell.execute_reply.started":"2024-09-26T18:09:05.467099Z","shell.execute_reply":"2024-09-26T18:09:05.525576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_columns:\n    if i not in columns:\n        print(i)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:03:53.517135Z","iopub.execute_input":"2024-09-26T18:03:53.517903Z","iopub.status.idle":"2024-09-26T18:03:53.527037Z","shell.execute_reply.started":"2024-09-26T18:03:53.517860Z","shell.execute_reply":"2024-09-26T18:03:53.526097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_and_preprocess_tabular_data(csv_path,columns=columns):\n    df = pd.read_csv(csv_path)\n    extras=[]\n    for i in df.columns:\n        if i not in columns:\n            extras.append(i)\n    extras.append('id')\n    # Separate labels (sii)\n    if 'sii' in df.columns:\n        y = df['sii']\n        X = df.drop(columns=extras)  # Remove 'id' and 'sii' for preprocessing\n    else:\n        y= None\n        X = df.drop(columns=['id'])\n    # Identify numeric and categorical columns\n    numeric_cols = X.select_dtypes(include=['float64', 'int64']).columns\n    categorical_cols = X.select_dtypes(include=['object']).columns\n\n    # Convert numeric columns to float (in case they contain string numbers)\n    X[numeric_cols] = X[numeric_cols].apply(pd.to_numeric, errors='coerce')\n\n    # Fill missing values: Numeric columns filled with mean, Categorical with mode\n    X[numeric_cols] = X[numeric_cols].fillna(X[numeric_cols].mean())\n    X[categorical_cols] = X[categorical_cols].fillna(X[categorical_cols].mode().iloc[0])\n    \n    # Label encode categorical columns\n    for col in categorical_cols:\n        X[col] = LabelEncoder().fit_transform(X[col].astype(str))  # Handle string types safely\n    \n    return X, y, df['id']","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:09.084994Z","iopub.execute_input":"2024-09-26T18:09:09.085819Z","iopub.status.idle":"2024-09-26T18:09:09.095093Z","shell.execute_reply.started":"2024-09-26T18:09:09.085776Z","shell.execute_reply":"2024-09-26T18:09:09.094128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scale_features(X_train, X_test=None, scaler=None):\n    # Create a new scaler if one is not provided (for training data)\n    if scaler is None:\n        scaler = StandardScaler()\n\n    # Fit and transform training data\n    X_train_scaled = scaler.fit_transform(X_train)\n\n    # Check if test data is provided, if so, transform it using the fitted scaler\n    if X_test is not None:\n        X_test_scaled = scaler.transform(X_test)\n        return X_train_scaled, X_test_scaled, scaler\n    else:\n        return X_train_scaled, None, scaler\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:12.574466Z","iopub.execute_input":"2024-09-26T18:09:12.575293Z","iopub.status.idle":"2024-09-26T18:09:12.580981Z","shell.execute_reply.started":"2024-09-26T18:09:12.575247Z","shell.execute_reply":"2024-09-26T18:09:12.579993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_time_series_data(series_folder, ids, max_timesteps=500):\n    time_series_data = []\n\n    for _id in ids:\n        series_path = os.path.join(series_folder, f\"id={_id}/part-0.parquet\")\n        if os.path.exists(series_path):\n            series_df = pd.read_parquet(series_path)\n            series_df.fillna(0, inplace=True)  # Fill NaN values in time-series data\n            \n            # Truncate or pad time series to the same length\n            truncated_series = series_df[['X', 'Y', 'Z', 'enmo', 'anglez','time_of_day','weekday','quarter']].values[:max_timesteps]\n            if truncated_series.shape[0] < max_timesteps:\n                padding = np.zeros((max_timesteps - truncated_series.shape[0], truncated_series.shape[1]))\n                truncated_series = np.vstack([truncated_series, padding])\n            time_series_data.append(truncated_series)\n        else:\n            # If no data for the ID, use all zeros\n            time_series_data.append(np.zeros((max_timesteps, 8)))  # Change to 5 for the correct number of columns\n\n    # Convert the list of arrays into a 3D NumPy array\n    return np.array(time_series_data)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:15.431529Z","iopub.execute_input":"2024-09-26T18:09:15.432309Z","iopub.status.idle":"2024-09-26T18:09:15.440407Z","shell.execute_reply.started":"2024-09-26T18:09:15.432264Z","shell.execute_reply":"2024-09-26T18:09:15.439200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import cohen_kappa_score\nimport numpy as np\n\nclass QWKCallback(Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        # Get the validation data\n        val_data = self.validation_data\n        val_pred = self.model.predict([val_data[0], val_data[1]])  # Tabular and Time-Series inputs\n        val_true = val_data[2]  # True validation labels\n        \n        # Get the predicted classes\n        val_pred_classes = np.argmax(val_pred, axis=-1)\n        \n        # Calculate QWK\n        qwk = cohen_kappa_score(val_true, val_pred_classes, weights=\"quadratic\")\n        \n        # Log QWK in the logs dictionary\n        print(f\"\\nEpoch {epoch + 1}: QWK = {qwk:.4f}\")\n        logs['qwk'] = qwk\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:18.245338Z","iopub.execute_input":"2024-09-26T18:09:18.246079Z","iopub.status.idle":"2024-09-26T18:09:18.255586Z","shell.execute_reply.started":"2024-09-26T18:09:18.246037Z","shell.execute_reply":"2024-09-26T18:09:18.254729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers, models, regularizers\n\ndef build_dual_head_model(input_shape_tabular, input_shape_series):\n    input_shape_tabular = (input_shape_tabular,)  # Convert scalar to tuple (e.g., (80,))\n    \n    # Tabular input head\n    input_tabular = layers.Input(shape=input_shape_tabular, name=\"tabular_input\")\n    x1 = layers.Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.001))(input_tabular)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.3)(x1)  # Add dropout to avoid overfitting\n    x1 = layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x1)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.3)(x1)\n    x1 = layers.Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x1)\n    x1 = layers.BatchNormalization()(x1)\n\n    # Time-series input head using LSTM\n    input_series = layers.Input(shape=input_shape_series, name=\"time_series_input\")\n    x2 = layers.LSTM(64, return_sequences=True)(input_series)  # Replacing Conv1D with LSTM\n    x2 = layers.BatchNormalization()(x2)\n    x2 = layers.Dropout(0.3)(x2)  # Add dropout to avoid overfitting\n    x2 = layers.LSTM(32)(x2)\n    x2 = layers.BatchNormalization()(x2)\n\n    # Concatenate both heads\n    concatenated = layers.concatenate([x1, x2])\n    x = layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.001))(concatenated)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    output = layers.Dense(4, activation='softmax')(x)  # Assuming 4-class classification\n\n    model = models.Model(inputs=[input_tabular, input_series], outputs=output)\n    \n    return model\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:22.012520Z","iopub.execute_input":"2024-09-26T18:09:22.013238Z","iopub.status.idle":"2024-09-26T18:09:22.024490Z","shell.execute_reply.started":"2024-09-26T18:09:22.013187Z","shell.execute_reply":"2024-09-26T18:09:22.023438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_loss(y_true, y_pred):\n    mask = tf.cast(tf.not_equal(y_true, -1), tf.float32)  # Ignore NaN labels (-1 in this case)\n    loss = tf.keras.losses.SparseCategoricalCrossentropy()(y_true, y_pred)\n    return tf.reduce_mean(loss * mask)\n\n# 6. Compile the model\ndef compile_model(model):\n    # Compile the model with RMSprop\n    opt = optimizers.Adam(learning_rate=1e-3)  # Try RMSprop\n    model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:26.099960Z","iopub.execute_input":"2024-09-26T18:09:26.100371Z","iopub.status.idle":"2024-09-26T18:09:26.106827Z","shell.execute_reply.started":"2024-09-26T18:09:26.100336Z","shell.execute_reply":"2024-09-26T18:09:26.105693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, X_tabular_train, X_series_train, y_train, epochs=20, batch_size=32):\n    history = model.fit([X_tabular_train, X_series_train], y_train, \n                        epochs=epochs, \n                        batch_size=batch_size, \n                        validation_split=0.2)\n    return history\n\n# 8. Inference and saving predictions\ndef predict_and_save(model, X_tabular_test, X_series_test, ids_test, output_csv):\n    predictions = model.predict([X_tabular_test, X_series_test])\n    predicted_labels = np.argmax(predictions, axis=1)\n    \n    # Create DataFrame for predictions\n    result_df = pd.DataFrame({\n        'id': ids_test,\n        'sii': predicted_labels\n    })\n    \n    # Save to CSV\n    result_df.to_csv(output_csv, index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:29.127324Z","iopub.execute_input":"2024-09-26T18:09:29.128010Z","iopub.status.idle":"2024-09-26T18:09:29.134866Z","shell.execute_reply.started":"2024-09-26T18:09:29.127966Z","shell.execute_reply":"2024-09-26T18:09:29.133791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tabular, y_train, ids_train = load_and_preprocess_tabular_data(train_csv_path)\n    \n    # Separate out labeled data (drop NaN labels)\nlabeled_mask = ~y_train.isna()\nX_tabular_train = X_tabular[labeled_mask]\ny_train = y_train[labeled_mask].astype(int)  # Drop NaNs in y_train\n    \n    # 2. Scale features\nX_tabular_train_scaled, _, scaler = scale_features(X_tabular_train)\n\n    # 3. Load corresponding time-series data\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:32.867967Z","iopub.execute_input":"2024-09-26T18:09:32.868641Z","iopub.status.idle":"2024-09-26T18:09:33.012551Z","shell.execute_reply.started":"2024-09-26T18:09:32.868598Z","shell.execute_reply":"2024-09-26T18:09:33.011498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_series_train = load_time_series_data(series_folder, ids_train[labeled_mask])\n\n    # 4. Prepare the shapes for the model\ninput_shape_tabular = X_tabular_train_scaled.shape[1]  # Number of tabular features\ninput_shape_series = X_series_train.shape[1:]  # Shape of time-series (timesteps, features)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:09:38.129839Z","iopub.execute_input":"2024-09-26T18:09:38.130959Z","iopub.status.idle":"2024-09-26T18:11:15.545168Z","shell.execute_reply.started":"2024-09-26T18:09:38.130911Z","shell.execute_reply":"2024-09-26T18:11:15.544053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_dual_head_model(input_shape_tabular, input_shape_series)\nmodel = compile_model(model)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:11:22.784344Z","iopub.execute_input":"2024-09-26T18:11:22.784774Z","iopub.status.idle":"2024-09-26T18:11:23.562335Z","shell.execute_reply.started":"2024-09-26T18:11:22.784735Z","shell.execute_reply":"2024-09-26T18:11:23.561265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check tabular data for NaN/Inf\nprint(f\"Tabular data has NaN: {np.any(np.isnan(X_tabular_train_scaled))}\")\nprint(f\"Tabular data has Inf: {np.any(np.isinf(X_tabular_train_scaled))}\")\n\n# Check time-series data for NaN/Inf\nprint(f\"Time-series data has NaN: {np.any(np.isnan(X_series_train))}\")\nprint(f\"Time-series data has Inf: {np.any(np.isinf(X_series_train))}\")\n\n# Check labels for NaN/Inf\nprint(f\"Labels have NaN: {np.any(np.isnan(y_train))}\")\nprint(f\"Labels have Inf: {np.any(np.isinf(y_train))}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:11:27.107615Z","iopub.execute_input":"2024-09-26T18:11:27.108575Z","iopub.status.idle":"2024-09-26T18:11:27.140017Z","shell.execute_reply.started":"2024-09-26T18:11:27.108520Z","shell.execute_reply":"2024-09-26T18:11:27.138802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tabular_train_scaled.max()","metadata":{"execution":{"iopub.status.busy":"2024-09-26T17:04:41.682188Z","iopub.execute_input":"2024-09-26T17:04:41.682602Z","iopub.status.idle":"2024-09-26T17:04:41.690093Z","shell.execute_reply.started":"2024-09-26T17:04:41.682565Z","shell.execute_reply":"2024-09-26T17:04:41.689165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.unique(y_train)) ","metadata":{"execution":{"iopub.status.busy":"2024-09-26T17:04:44.083878Z","iopub.execute_input":"2024-09-26T17:04:44.084468Z","iopub.status.idle":"2024-09-26T17:04:44.090008Z","shell.execute_reply.started":"2024-09-26T17:04:44.084429Z","shell.execute_reply":"2024-09-26T17:04:44.088977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = train_model(model, X_tabular_train_scaled, X_series_train, y_train, epochs=20, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:11:32.137028Z","iopub.execute_input":"2024-09-26T18:11:32.137460Z","iopub.status.idle":"2024-09-26T18:12:47.028559Z","shell.execute_reply.started":"2024-09-26T18:11:32.137414Z","shell.execute_reply":"2024-09-26T18:12:47.027617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tabular_test, _, ids_test = load_and_preprocess_tabular_data(test_csv_path)  # No sii column here\nX_tabular_test_scaled, _, _ = scale_features(X_tabular_test, scaler=scaler)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:12:52.362524Z","iopub.execute_input":"2024-09-26T18:12:52.362974Z","iopub.status.idle":"2024-09-26T18:12:52.436301Z","shell.execute_reply.started":"2024-09-26T18:12:52.362920Z","shell.execute_reply":"2024-09-26T18:12:52.435239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_series_test = load_time_series_data('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet', ids_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:12:58.090970Z","iopub.execute_input":"2024-09-26T18:12:58.091385Z","iopub.status.idle":"2024-09-26T18:12:58.289709Z","shell.execute_reply.started":"2024-09-26T18:12:58.091345Z","shell.execute_reply":"2024-09-26T18:12:58.288803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_and_save(model, X_tabular_test_scaled, X_series_test, ids_test, 'submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-26T18:13:02.507420Z","iopub.execute_input":"2024-09-26T18:13:02.508468Z","iopub.status.idle":"2024-09-26T18:13:02.859172Z","shell.execute_reply.started":"2024-09-26T18:13:02.508421Z","shell.execute_reply":"2024-09-26T18:13:02.858171Z"},"trusted":true},"execution_count":null,"outputs":[]}]}