{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9732360,"sourceType":"datasetVersion","datasetId":5955980},{"sourceId":9797711,"sourceType":"datasetVersion","datasetId":5994517},{"sourceId":9798490,"sourceType":"datasetVersion","datasetId":6005036},{"sourceId":10000596,"sourceType":"datasetVersion","datasetId":6148646},{"sourceId":10013982,"sourceType":"datasetVersion","datasetId":6165256}],"dockerImageVersionId":30775,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Child Mind Institute — Problematic Internet Use\n**Relating Physical Activity to Problematic Internet Use**\n\nThe goal of this competition is to develop a predictive model that analyzes children's physical activity and fitness data to identify early signs of problematic internet use. Identifying these patterns can help trigger interventions to encourage healthier digital habits.\n\n**Credits**\n\nThis solution was based on some sophisticated DNN that was the winning solution of the <a href=\"https://www.kaggle.com/competitions/icr-identify-age-related-conditions\">ICR competition</a> \n\n<a href=\"https://www.kaggle.com/competitions/icr-identify-age-related-conditions/discussion/430843\">Here</a> the solution explained by its author @ROOM722\n\nHis solution, in turn, was inspired by the following Keras code example:\n<a href=\"https://keras.io/examples/structured_data/classification_with_grn_and_vsn/\">Classification with Gated Residual and Variable Selection Networks</a>\n\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nimport os\nimport random\nimport gc\nimport pathlib\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score, confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.utils import class_weight\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.linear_model import BayesianRidge, Ridge\nfrom scipy import stats as st\n\nimport tensorflow as tf\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, Callback\nfrom tensorflow.keras import regularizers as R\nfrom tensorflow.keras.models import Model, load_model, Sequential\nfrom tensorflow.keras import layers as L\nfrom tensorflow.keras import optimizers as O\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.metrics import CategoricalCrossentropy\nfrom tensorflow.keras.optimizers import Adam","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:02.120403Z","iopub.execute_input":"2024-12-05T00:13:02.120866Z","iopub.status.idle":"2024-12-05T00:13:19.285346Z","shell.execute_reply.started":"2024-12-05T00:13:02.120829Z","shell.execute_reply":"2024-12-05T00:13:19.284252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SAVE_MY_TRAIN = False\nUSE_MY_TRAIN = False\nFULL_TRAIN = True\nVALID = False\nTEST = True  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.287223Z","iopub.execute_input":"2024-12-05T00:13:19.287840Z","iopub.status.idle":"2024-12-05T00:13:19.293072Z","shell.execute_reply.started":"2024-12-05T00:13:19.287804Z","shell.execute_reply":"2024-12-05T00:13:19.291782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_randoms(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.294238Z","iopub.execute_input":"2024-12-05T00:13:19.294584Z","iopub.status.idle":"2024-12-05T00:13:19.319396Z","shell.execute_reply.started":"2024-12-05T00:13:19.294552Z","shell.execute_reply":"2024-12-05T00:13:19.318227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nset_randoms(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.321712Z","iopub.execute_input":"2024-12-05T00:13:19.322087Z","iopub.status.idle":"2024-12-05T00:13:19.335257Z","shell.execute_reply.started":"2024-12-05T00:13:19.322054Z","shell.execute_reply":"2024-12-05T00:13:19.334226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.options.display.max_rows = None\npd.options.display.max_columns = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.336581Z","iopub.execute_input":"2024-12-05T00:13:19.336899Z","iopub.status.idle":"2024-12-05T00:13:19.349847Z","shell.execute_reply.started":"2024-12-05T00:13:19.336870Z","shell.execute_reply":"2024-12-05T00:13:19.348693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# ideas taken from the following notebooks:\n# https://www.kaggle.com/code/abdmental01/cmi-best-single-model\n# https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-actigraphy-data-eda\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df = df[df['non-wear_flag'] == 0]\n    \n    df['time_of_day_hours'] = (df['time_of_day'] / 1e9 / 3600)  # nanoseconds to hours\n        \n    stats = np.full((4, 6), fill_value=-1, dtype='float32') \n    stats_business_day = np.full((4, 3), fill_value=-1, dtype='float32') \n    stats_weekend = np.full((4, 3), fill_value=-1, dtype='float32') \n    for i in range(4):\n        stats_business_day[i] = df[(df['weekday'].isin([1, 2, 3, 4, 5])) & (df['time_of_day_hours'] >= i * 6) & (df['time_of_day_hours'] < (i + 1) * 6)][['enmo']].agg(['mean', 'max', 'std']).values.reshape(-1)\n        stats_weekend[i] = df[(df['weekday'].isin([6, 7])) & (df['time_of_day_hours'] >= i * 6) & (df['time_of_day_hours'] < (i + 1) * 6)][['enmo']].agg(['mean','max', 'std']).values.reshape(-1)\n        stats[i] = np.concatenate([stats_business_day[i], stats_weekend[i]])\n\n    return stats.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.351247Z","iopub.execute_input":"2024-12-05T00:13:19.351687Z","iopub.status.idle":"2024-12-05T00:13:19.368719Z","shell.execute_reply.started":"2024-12-05T00:13:19.351642Z","shell.execute_reply":"2024-12-05T00:13:19.367639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_df(df):\n    df.loc[df['Physical-BMI'] == 0, 'Physical-BMI'] = None\n    df.loc[df['Physical-Weight'] == 0, 'Physical-Weight'] = None\n    df.loc[df['Physical-Diastolic_BP'] == 0, 'Physical-Diastolic_BP'] = None\n    df.loc[df['Physical-Systolic_BP'] == 0, 'Physical-Systolic_BP'] = None\n    \n    seasons = {'Winter': 0, 'Fall': 1, 'Spring': 2, 'Summer': 3}\n    season_cols = [c for c in df.columns if 'Season' in c]\n    for c in season_cols:\n        df[c] = df[c].map(seasons)\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.370063Z","iopub.execute_input":"2024-12-05T00:13:19.370408Z","iopub.status.idle":"2024-12-05T00:13:19.384825Z","shell.execute_reply.started":"2024-12-05T00:13:19.370377Z","shell.execute_reply":"2024-12-05T00:13:19.383804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv', index_col='id')\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntrain_df = pd.merge(train_df, train_ts, how=\"left\", on='id').set_index('id')\ntrain_df = preprocess_df(train_df)\n\n# some data corrections\ntrain_df.drop(['83525bbe'], axis=0, inplace=True)\ntrain_df.loc['464a75fb', 'Physical-Diastolic_BP'] = 79.0\ntrain_df.loc[['967d790c', '77a5b2ad'], 'Physical-Systolic_BP'] = None\ntrain_df.loc['d819548d', 'Physical-Systolic_BP'] = 173.0\ntrain_df.loc['f2499682', 'Fitness_Endurance-Time_Mins'] = None\n\ntrain_df.drop(columns=[c for c in train_df.columns if c.startswith('PCIAT-')], inplace=True)\n\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv', index_col='id')\ntest_df = pd.merge(test_df, test_ts, how=\"left\", on='id').set_index('id')\ntest_df = preprocess_df(test_df)\n\nseason_cols = [c for c in train_df.columns if 'Season' in c]\nfeatures_to_transform = [f for f in train_df.columns if f not in season_cols and f != 'sii']\nscaler = MinMaxScaler()\nscaler.fit(pd.concat([train_df[features_to_transform], test_df[features_to_transform]], ignore_index=True))\n# scaler.fit(train_df[features_to_transform])\ntrain_df[features_to_transform] = scaler.transform(train_df[features_to_transform])\ntest_df[features_to_transform] = scaler.transform(test_df[features_to_transform])\n\nfeatures_to_impute = [f for f in train_df.columns if f != 'sii']\nimputer = IterativeImputer(estimator=BayesianRidge(), max_iter=25, \n                           random_state=0)   \nimputer.fit(pd.concat([train_df[features_to_impute], test_df[features_to_impute]], ignore_index=True))\n# imputer.fit(train_df[features_to_impute])\ntrain_df_filled = pd.DataFrame(imputer.transform(train_df[features_to_impute]), columns=features_to_impute, \n                                                         index=train_df.index)\ntrain_df_filled['sii'] = train_df['sii']\ntrain_df = train_df_filled.dropna(subset=['sii']).reset_index()\n\nif TEST:\n    test_df_filled = pd.DataFrame(imputer.transform(test_df), columns=test_df.columns, index=test_df.index)\n    test_df = test_df_filled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:13:19.386920Z","iopub.execute_input":"2024-12-05T00:13:19.387249Z","iopub.status.idle":"2024-12-05T00:17:34.525961Z","shell.execute_reply.started":"2024-12-05T00:13:19.387218Z","shell.execute_reply":"2024-12-05T00:17:34.524434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.528130Z","iopub.execute_input":"2024-12-05T00:17:34.528534Z","iopub.status.idle":"2024-12-05T00:17:34.756079Z","shell.execute_reply.started":"2024-12-05T00:17:34.528496Z","shell.execute_reply":"2024-12-05T00:17:34.754721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@tf.keras.utils.register_keras_serializable()\ndef smish(x):\n    return x * K.tanh(K.log(1 + K.sigmoid(x)))\n\n\n@tf.keras.utils.register_keras_serializable()\nclass GatedLinearUnit(L.Layer):\n    def __init__(self, units, **kwargs):\n        super().__init__(**kwargs)\n        self.linear = L.Dense(units)\n        self.sigmoid = L.Dense(units, activation=\"sigmoid\")\n        self.units = units\n\n    def get_config(self):\n        config = super().get_config()\n        config['units'] = self.units\n        return config\n    \n    def call(self, inputs):\n        return self.linear(inputs) * self.sigmoid(inputs)\n    \n\n@tf.keras.utils.register_keras_serializable()\nclass GatedResidualNetwork(L.Layer):\n    def __init__(self, units, dropout_rate, **kwargs):\n        super().__init__(**kwargs)\n        self.units = units\n        self.dropout_rate = dropout_rate\n        self.relu_dense = L.Dense(units, activation=smish)\n        self.linear_dense = L.Dense(units)\n        self.dropout = L.Dropout(dropout_rate)\n        self.gated_linear_unit = GatedLinearUnit(units)\n        self.layer_norm = L.LayerNormalization()\n        self.project = L.Dense(units)\n\n    def get_config(self):\n        config = super().get_config()\n        config['units'] = self.units\n        config['dropout_rate'] = self.dropout_rate\n        return config\n    \n    def call(self, inputs):\n        x = self.relu_dense(inputs)\n        x = self.linear_dense(x)\n        x = self.dropout(x)\n        if inputs.shape[-1] != self.units:\n            inputs = self.project(inputs)\n        x = inputs + self.gated_linear_unit(x)\n        x = self.layer_norm(x)\n        return x\n    \n\n@tf.keras.utils.register_keras_serializable()\nclass VariableSelection(L.Layer):\n    def __init__(self, num_features, units, dropout_rate, **kwargs):\n        super().__init__(**kwargs)\n        self.grns = list()\n        # Create a GRN for each feature independently\n        for idx in range(num_features):\n            grn = GatedResidualNetwork(units, dropout_rate)\n            self.grns.append(grn)\n        # Create a GRN for the concatenation of all the features\n        self.grn_concat = GatedResidualNetwork(units, dropout_rate)\n        self.softmax = L.Dense(units=num_features, activation=\"softmax\")\n        self.num_features = num_features\n        self.units = units\n        self.dropout_rate = dropout_rate\n\n    def get_config(self):\n        config = super().get_config()\n        config['num_features'] = self.num_features\n        config['units'] = self.units\n        config['dropout_rate'] = self.dropout_rate\n        return config\n    \n    def call(self, inputs):\n        v = L.concatenate(inputs)\n        v = self.grn_concat(v)\n        v = tf.expand_dims(self.softmax(v), axis=-1)\n\n        x = []\n        for idx, input_ in enumerate(inputs):\n            x.append(self.grns[idx](input_))\n        x = tf.stack(x, axis=1)\n\n        outputs = tf.squeeze(tf.matmul(v, x, transpose_a=True), axis=1)\n        return outputs\n    \n\n@tf.keras.utils.register_keras_serializable()\nclass VariableSelectionFlow(L.Layer):\n    def __init__(self, num_features, units, dropout_rate, dense_units=None, **kwargs):\n        super().__init__(**kwargs)\n        self.variableselection = VariableSelection(num_features, units, dropout_rate)\n        self.split = L.Lambda(lambda t: tf.split(t, num_features, axis=-1))\n        self.dense = dense_units\n        if dense_units:\n            self.dense_list = [L.Dense(dense_units, \\\n                                       activation='linear') \\\n                               for _ in tf.range(num_features)\n                              ]\n        self.num_features = num_features\n        self.units = units\n        self.dropout_rate = dropout_rate\n        self.dense_units = dense_units\n        \n    def get_config(self):\n        config = super().get_config()\n        config['num_features'] = self.num_features\n        config['units'] = self.units\n        config['dropout_rate'] = self.dropout_rate\n        config['dense_units'] = self.dense_units\n        return config        \n    \n    def call(self, inputs):\n        split_input = self.split(inputs)\n        if self.dense:\n            l = [self.dense_list[i](split_input[i]) for i in range(len(self.dense_list))]\n        else:\n            l = split_input\n        return self.variableselection(l)    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.761822Z","iopub.execute_input":"2024-12-05T00:17:34.762190Z","iopub.status.idle":"2024-12-05T00:17:34.786851Z","shell.execute_reply.started":"2024-12-05T00:17:34.762159Z","shell.execute_reply":"2024-12-05T00:17:34.785738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 10\nbatch_size = 32\n\nunits_1 = 32\ndrop_1 = 0.75\ndense_units = 0 # 8\n\nunits_2 = 16\ndrop_2 = 0.5\n\nunits_3 = 8\ndrop_3 = 0.25","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.788307Z","iopub.execute_input":"2024-12-05T00:17:34.788704Z","iopub.status.idle":"2024-12-05T00:17:34.806810Z","shell.execute_reply.started":"2024-12-05T00:17:34.788669Z","shell.execute_reply":"2024-12-05T00:17:34.805217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_rate = 0.001\n\nnum_epochs = 10 if FULL_TRAIN else 10 # 50   \n\ndef get_model():\n    inputs_1 = tf.keras.Input(shape=(82,))  \n        \n    features_1 = VariableSelectionFlow(82, units_1, drop_1, dense_units=dense_units)(inputs_1)\n    features_2 = VariableSelectionFlow(units_1, units_2, drop_2)(features_1)         \n    features_3 = VariableSelectionFlow(units_2, units_3, drop_3)(features_2)         \n\n    outputs = L.Dense(4, activation=\"softmax\")(features_3)\n    \n    model = Model(inputs=inputs_1, outputs=outputs)\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.808306Z","iopub.execute_input":"2024-12-05T00:17:34.808745Z","iopub.status.idle":"2024-12-05T00:17:34.819731Z","shell.execute_reply.started":"2024-12-05T00:17:34.808692Z","shell.execute_reply":"2024-12-05T00:17:34.818651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_categorical_crossentropy(class_weight):\n    def loss(y_true, y_pred):\n        y_true = tf.dtypes.cast(y_true, tf.float64) \n        sample_weight = tf.math.multiply(class_weight, y_true)\n        sample_weight = tf.reduce_sum(sample_weight, axis=-1)     \n        cce = tf.keras.losses.CategoricalCrossentropy()\n        losses = cce(y_true, y_pred, sample_weight=sample_weight)\n        return losses\n    \n    return loss    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.821039Z","iopub.execute_input":"2024-12-05T00:17:34.821484Z","iopub.status.idle":"2024-12-05T00:17:34.834998Z","shell.execute_reply.started":"2024-12-05T00:17:34.821437Z","shell.execute_reply":"2024-12-05T00:17:34.833534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ordinal_weighted_categorical_crossentropy(class_weight):\n    def loss(y_true, y_pred):\n        ord_weights = K.cast(K.abs(K.argmax(y_true, axis=1) - K.argmax(y_pred, axis=1)) / (K.int_shape(y_pred)[1] - 1), dtype='float32')\n        \n        y_true = tf.dtypes.cast(y_true, tf.float64) \n        sample_weight = tf.math.multiply(class_weight, y_true)\n        sample_weight = tf.reduce_sum(sample_weight, axis=-1)     \n        cce = tf.keras.losses.CategoricalCrossentropy()\n        losses = cce(y_true, y_pred, sample_weight=sample_weight)  \n        alpha = 0.8\n        return (1.0 + ord_weights * alpha) * losses\n    \n    return loss    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.836492Z","iopub.execute_input":"2024-12-05T00:17:34.836932Z","iopub.status.idle":"2024-12-05T00:17:34.858553Z","shell.execute_reply.started":"2024-12-05T00:17:34.836899Z","shell.execute_reply":"2024-12-05T00:17:34.857292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_MY_TRAIN:\n    train_df = pd.read_csv('/kaggle/input/piu-train/my_train.csv')    \n    train_df['sii_with_difficulty'] = train_df['sii']\n    train_df.loc[train_df['score_real_sii'] < 0.15, 'sii_with_difficulty'] = train_df['sii'] + 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.860223Z","iopub.execute_input":"2024-12-05T00:17:34.860626Z","iopub.status.idle":"2024-12-05T00:17:34.873118Z","shell.execute_reply.started":"2024-12-05T00:17:34.860591Z","shell.execute_reply":"2024-12-05T00:17:34.871810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['sii'] = train_df['sii'].astype(int)\nX_train_size = train_df.shape[0]\n\nif VALID:\n    oof_preds = np.zeros((X_train_size, 4))\n    oof_preds_sii = np.zeros(X_train_size)\n    folds = np.zeros(X_train_size)\n\nif TEST:\n    preds_test = np.zeros((n_splits, test_df.shape[0], 4))\n    preds_test_sii = np.zeros(test_df.shape[0])   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.874661Z","iopub.execute_input":"2024-12-05T00:17:34.875122Z","iopub.status.idle":"2024-12-05T00:17:34.887729Z","shell.execute_reply.started":"2024-12-05T00:17:34.875073Z","shell.execute_reply":"2024-12-05T00:17:34.886289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if VALID:\n    my_train_df = pd.read_csv('/kaggle/input/piu-train/my_train.csv')\n    train_df['sii_with_difficulty'] = my_train_df['sii']\n    train_df['score_real_sii'] = my_train_df['score_real_sii']\n    train_df.loc[train_df['score_real_sii'] < 0.15, 'sii_with_difficulty'] = train_df['sii'] + 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:34.889974Z","iopub.execute_input":"2024-12-05T00:17:34.890627Z","iopub.status.idle":"2024-12-05T00:17:35.050790Z","shell.execute_reply.started":"2024-12-05T00:17:34.890585Z","shell.execute_reply":"2024-12-05T00:17:35.049185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if FULL_TRAIN:\n    n_models = 15\n    preds_test_models = np.zeros((n_models, test_df.shape[0], 4))\n\n    if VALID:\n        X_train = train_df.drop(['id', 'sii', 'score_real_sii', 'sii_with_difficulty'], axis=1)\n    else:\n        X_train = train_df.drop(['id', 'sii'], axis=1)\n    y_train = pd.get_dummies(train_df['sii'], dtype=float)\n    \n    class_weights = np.array([0.06, 0.12, 0.18, 0.64])\n    wcc = weighted_categorical_crossentropy(class_weights)   \n    # wcc = ordinal_weighted_categorical_crossentropy(class_weights)\n      \n    for i in range(n_models):  \n        if n_models > 1:\n            set_randoms(i)\n\n        model = get_model()\n        model.compile(\n                optimizer=Adam(learning_rate=learning_rate, epsilon=1e-7),  \n                loss=wcc\n            ) \n\n        print(f\"Start training the model {i} ...\")\n        model.fit(\n                x=X_train,\n                y=y_train,\n                batch_size=batch_size,\n                epochs=num_epochs,\n                shuffle=True\n            )\n        print(f\"Model training finished {i}.\")\n\n        if TEST:\n            preds_test_models[i] = model.predict(test_df.values) \n            \n    if TEST:\n        preds_test_full_train = np.mean(preds_test_models, axis=0)\n        preds_test_sii_full_train = np.argmax(preds_test_full_train, axis=1)\n\n    preds_train = model.predict(X_train)\n    y_train_sii = np.argmax(y_train, axis=1)\n    preds_sii_train = np.argmax(preds_train, axis=1)\n    res_train = cohen_kappa_score(y_train_sii, preds_sii_train, weights=\"quadratic\")\n    print('TRAIN_RES = ', res_train, '\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T00:17:35.052324Z","iopub.execute_input":"2024-12-05T00:17:35.052768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if VALID:\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    for fold,(train_idx, valid_idx) in enumerate(SKF.split(train_df, train_df['sii_with_difficulty'])):\n        \n        set_randoms(27)  \n\n        print('#'*25)\n        print('### Fold',fold + 1)\n        print('### Train size', train_idx.shape[0], 'Valid size', valid_idx.shape[0])\n        print('#'*25)\n\n        X_train = train_df.drop(['id', 'sii', 'score_real_sii', 'sii_with_difficulty'], axis=1).loc[train_idx]\n        y_train = pd.get_dummies(train_df.loc[train_idx,'sii'], dtype=float)\n        X_valid = train_df.drop(['id', 'sii', 'score_real_sii', 'sii_with_difficulty'], axis=1).loc[valid_idx]\n        y_valid = pd.get_dummies(train_df.loc[valid_idx, 'sii'], dtype=float)\n\n        class_weights = np.array([0.06, 0.12, 0.18, 0.64])\n        # wcc = weighted_categorical_crossentropy(class_weights)  # Sacar\n        wcc = ordinal_weighted_categorical_crossentropy(class_weights)\n\n        model = get_model()\n\n        model.compile(\n            optimizer=Adam(learning_rate=learning_rate, epsilon=1e-7),  \n            loss=wcc\n        )\n\n        # Create an early stopping callback\n        early_stopping = tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_loss\", patience=10, restore_best_weights=True     \n         )\n        lr = ReduceLROnPlateau(monitor=\"val_loss\", mode='min', factor=0.95, patience=1, verbose=0) \n\n        print(\"Start training the model...\")\n        history = model.fit(\n            x=X_train,\n            y=y_train,\n            batch_size=batch_size,\n            epochs=num_epochs,\n            shuffle=True,\n            validation_data=(X_valid, y_valid),\n            callbacks=[lr]\n            # callbacks=[lr, early_stopping]   \n        )\n        print(\"Model training finished.\")\n        val_loss_hist = history.history['val_loss']\n        print(f'best epoch: {np.argmin(val_loss_hist)}')\n\n        if VALID:\n            preds = model.predict(X_valid)\n\n            y_valid_sii = np.argmax(y_valid, axis=1)\n            preds_sii = np.argmax(preds, axis=1)\n\n            preds_train = model.predict(X_train)\n            y_train_sii = np.argmax(y_train, axis=1)\n            preds_sii_train = np.argmax(preds_train, axis=1)\n            res_train = cohen_kappa_score(y_train_sii, preds_sii_train, weights=\"quadratic\")\n            print('TRAIN_RES = ', res_train, '\\n')\n            \n            res = cohen_kappa_score(y_valid_sii, preds_sii, weights=\"quadratic\")\n            print('RES = ', res, '\\n')\n\n            oof_preds[valid_idx] = preds\n            oof_preds_sii[valid_idx] = preds_sii\n            folds[valid_idx] = fold\n\n        if TEST:\n            preds_test[fold] = model.predict(test_df.values)  \n            \n        model.save_weights(f'mod_v64_f{fold}_res{res:.4f}.weights.h5')\n        \n    if TEST:\n        preds_test_cv = np.mean(preds_test, axis=0)\n        preds_test_sii_cv = np.argmax(preds_test_cv, axis=1)\n\n    print('#'*25)\n    res = cohen_kappa_score(train_df.sii, oof_preds_sii, weights=\"quadratic\")\n    print('OVERALL RES =', res,'\\n')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if SAVE_MY_TRAIN:\n    train_df['score_real_sii'] = None\n    for i in range(train_df.shape[0]):\n        train_df.loc[i, 'score_real_sii'] = oof_preds[i, train_df.loc[i, 'sii']]\n    train_df.to_csv('my_train.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if VALID:\n    tot = np.zeros(4)\n    tot_real = np.zeros(4)\n    for i in range(oof_preds_sii.shape[0]):\n        tot[int(oof_preds_sii[i])] += 1\n        tot_real[int(train_df.loc[i, 'sii'])] += 1\n    print(f'TOT REAL: {tot_real}')\n    print(f'TOT PRED: {tot}')\n\n    oof_preds_sii = oof_preds_sii.astype(int)\n\n    c = confusion_matrix(train_df['sii'].values, oof_preds_sii)\n    cm_display = ConfusionMatrixDisplay(confusion_matrix=c, display_labels=[0, 1, 2, 3])\n    cm_display.plot()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TEST:\n    if FULL_TRAIN and VALID:\n        preds_test = 0.5 * preds_test_cv + 0.5 * preds_test_full_train\n    elif FULL_TRAIN:\n        preds_test = preds_test_full_train\n    else:\n        preds_test = preds_test_cv\n    preds_test_sii = np.argmax(preds_test, axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TEST and FULL_TRAIN and VALID:\n    res = cohen_kappa_score(preds_test_sii_full_train, preds_test_sii_cv, weights=\"quadratic\")\n    print('KAPPA FULL_TRAIN - CV = ', res,'\\n')\n    \n    res = cohen_kappa_score(preds_test_sii_full_train, preds_test_sii, weights=\"quadratic\")\n    print('KAPPA FULL_TRAIN - PRED = ', res,'\\n')\n\n    res = cohen_kappa_score(preds_test_sii_cv, preds_test_sii, weights=\"quadratic\")\n    print('KAPPA CV - PRED = ', res,'\\n')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TEST:\n    test_df['sii'] = preds_test_sii\n    test_df['sii'].to_csv('submission.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head(20)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!zip files.zip *\n\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\n\nFileLink(r'files.zip')","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}