{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport os\nimport numpy as np\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, cohen_kappa_score, mean_absolute_error\nfrom sklearn.preprocessing import MinMaxScaler\nimport tensorflow as tf\nimport pickle\nfrom scipy.optimize import minimize\nimport lightgbm as lgb\nfrom catboost import CatBoost, Pool\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nif tf.config.list_physical_devices('GPU') :\n    device_name=tf.test.gpu_device_name()\nelse :\n    device_name='/CPU:0'\nprint(device_name)\n\nrandom_seed = 42\nos.environ['PYTHONHASHSEED'] = '0'\nos.environ['TF_DETERMINISTIC_OPS'] = '1'\nnp.random.seed(random_seed)\nrandom.seed(random_seed)\ntf.random.set_seed(random_seed)\n\ndir = '/kaggle/input/child-mind-institute-problematic-internet-use'\ndf_train = pl.read_csv(os.path.join(dir, 'train.csv'))\ndf_test = pl.read_csv(os.path.join(dir, 'test.csv'))\n\ndef preprocess(dir, train_data, test_data=pl.DataFrame()) :\n\n    if test_data.is_empty() == False :\n        df_data = test_data\n    else :\n        df_data = train_data \n        PCIAT_cols = [c for c in df_data.columns if 'PCIAT' in c]\n        df_data = df_data.drop(PCIAT_cols)\n\n    df_data = df_data.with_columns(df_data.transpose().null_count().transpose().to_series().alias('null_count'))\n\n    scaler = MinMaxScaler()\n    \n    Physical_cols = [c for c in train_data.columns if 'Physical' in c][1:]\n    scaler.fit(train_data[Physical_cols])\n    physical = scaler.transform(df_data[Physical_cols])\n    df_data = df_data.with_columns(physical_mean=np.nanmean(physical, axis=1),\n                                   physical_std=np.nanstd(physical, axis=1))\n    df_data = df_data.with_columns(pl.when(pl.col('physical_mean')==np.nan).then(None)\n                                     .otherwise(pl.col('physical_mean')).alias('physical_mean_n'),\n                                   pl.when(pl.col('physical_std')==np.nan).then(None)\n                                     .otherwise(pl.col('physical_std')).alias('physical_std_n')   \n                                  ).drop('physical_mean', 'physical_std')\n    FGC_cols = [c for c in train_data.columns if 'FGC' in c][1:]\n    scaler.fit(train_data[FGC_cols])\n    fgc = scaler.transform(df_data[FGC_cols])\n    df_data = df_data.with_columns(fgc_mean=np.nanmean(fgc, axis=1),\n                                   fgc_std=np.nanstd(fgc, axis=1))\n    df_data = df_data.with_columns(pl.when(pl.col('fgc_mean')==np.nan).then(None)\n                                     .otherwise(pl.col('fgc_mean')).alias('fgc_mean_n'),\n                                   pl.when(pl.col('fgc_std')==np.nan).then(None)\n                                     .otherwise(pl.col('fgc_std')).alias('fgc_std_n')   \n                                  ).drop('fgc_mean', 'fgc_std')\n    BIA_cols = [c for c in train_data.columns if 'BIA' in c][1:]\n    scaler.fit(train_data[BIA_cols])\n    bia = scaler.transform(df_data[BIA_cols])\n    df_data = df_data.with_columns(bia_mean=np.nanmean(bia, axis=1),\n                                   bia_std=np.nanstd(bia, axis=1))\n    df_data = df_data.with_columns(pl.when(pl.col('bia_mean')==np.nan).then(None)\n                                     .otherwise(pl.col('bia_mean')).alias('bia_mean_n'),\n                                   pl.when(pl.col('bia_std')==np.nan).then(None)\n                                     .otherwise(pl.col('bia_std')).alias('bia_std_n')   \n                                  ).drop('bia_mean', 'bia_std')\n\n    if test_data.is_empty() == False :\n        parquet_name = 'series_test.parquet'    \n    else :\n        parquet_name = 'series_train.parquet'\n\n    dir_series_train = os.listdir(os.path.join(dir, parquet_name))\n    part_0_train = pl.DataFrame()\n    for d in dir_series_train :\n        series_train_id = d.split('=') \n        path = os.path.join(dir, parquet_name, d)\n        df_series_train = pl.read_parquet(os.path.join(path, 'part-0.parquet'))\n        non_wear = df_series_train.filter(df_series_train['non-wear_flag']>0.5)\n        non_wear_ratio = len(non_wear) / len(df_series_train)\n        part_0_train_sub = pl.DataFrame({'id':series_train_id[-1],\n                                         'len_series_train':len(df_series_train),\n                                         'non_wear_ratio':non_wear_ratio})\n        df_series_train = df_series_train.filter(df_series_train['non-wear_flag']<=0.5)\n        df_series_train = df_series_train[['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'relative_date_PCIAT']]\n        for c in df_series_train.columns[:-1] :\n            part_0_train_sub = part_0_train_sub.with_columns(pl.DataFrame({c+'_mean':df_series_train[c].mean(),\n                                                                           c+'_std':df_series_train[c].std(),\n                                                                           c+'_skew':df_series_train[c].skew()}))\n        part_0_train_sub = part_0_train_sub.with_columns(relative_date_min=df_series_train['relative_date_PCIAT'].min(),\n                                                         relative_date_max=df_series_train['relative_date_PCIAT'].max())                                                                              \n        part_0_train = pl.concat([part_0_train, part_0_train_sub])\n\n    df_id = df_data['id']\n    df_data = df_data.join(part_0_train, on='id', how='left').drop('id')\n\n    str_cols = df_data.select(pl.col(pl.Utf8)).columns\n    season = df_data[str_cols[-1]].unique().sort()\n    df_data = df_data.with_columns(df_data[str_cols[-1]].str.replace(season[1], 0)\n                                                            .replace(season[2], 1)\n                                                            .replace(season[3], 2)\n                                                            .replace(season[4], 3).cast(pl.Int32).alias('season'))\n    for c in str_cols :\n        if c == str_cols[-1] :\n            df_data = df_data.drop('season')\n            break\n        else :\n            df_data = df_data.with_columns(df_data[c].str.replace(season[1], 0)\n                                                         .replace(season[2], 1)\n                                                         .replace(season[3], 2)\n                                                         .replace(season[4], 3).cast(pl.Int32).alias(c+'_int'))\n            df_data = df_data.with_columns((pl.when((df_data['season']-df_data[c+'_int']).is_null()==True).then(None)\n                                              .otherwise(pl.when(df_data['season']-df_data[c+'_int']==0).then(1)\n                                              .otherwise(0))).alias(c+'_diff')).drop(c, c+'_int')\n\n    str_cols = df_data.select(pl.col(pl.Utf8)).columns\n    df_data = df_data.with_columns(df_data[str_cols].to_dummies()).drop(str_cols)\n\n    mean_cols = [c for c in df_data.columns if 'sii' not in c and 'null_count' not in c and 'PCIAT-PCIAT_Total' not in c]\n    age_sex_mean = df_data[mean_cols].group_by('Basic_Demos-Age', 'Basic_Demos-Sex').agg(pl.col(mean_cols[2:]).mean())\n    for c in mean_cols[2:] :\n        age_sex_mean = age_sex_mean.with_columns(pl.when(pl.col(c)==np.nan).then(None).otherwise(pl.col(c))\n                                                .alias(c+'_age_sex_mean')).drop(c)\n    age_sex_mean = age_sex_mean.fill_null(strategy='mean')\n\n    df_data = df_data.join(age_sex_mean, on=('Basic_Demos-Age', 'Basic_Demos-Sex'), how='left')\n\n    for c in mean_cols[2:] :\n        df_data = df_data.with_columns(pl.when(pl.col(c).is_null()).then(pl.col(c+'_age_sex_mean'))\n                                         .otherwise(pl.col(c)).alias(c+'_r')).drop(c, c+'_age_sex_mean')\n\n    if test_data.is_empty() == False :\n        df_data = df_data.fill_null(strategy='mean')\n        with open('scaler.pkl', 'rb') as f :\n            scaler = pickle.load(f)\n        df_inp = scaler.transform(df_data)\n        df_tar = None\n    else :\n        df_data = df_data.drop_nulls(subset='sii')\n        df_inp = df_data.drop('sii')\n        scaler.fit(df_inp)\n        df_inp = scaler.transform(df_inp)\n        with open('scaler.pkl', 'wb') as f :\n            pickle.dump(scaler, f)\n        df_tar = df_data['sii'].to_numpy()\n\n    df_inp = df_inp[:, :, np.newaxis]\n\n    return df_inp, df_tar, df_id        \n\ndef mlp(x, hidden_units, dropout_rate) :\n    for units in hidden_units :\n        x = tf.keras.layers.Dense(units, activation=tf.nn.gelu)(x)\n        x = tf.keras.layers.Dropout(dropout_rate)(x)\n    return x\n\nclass Encoder(tf.keras.layers.Layer) :\n    def __init__(self, input_dim, projection_dim) :\n        super(Encoder, self).__init__()\n        self.input_dim = input_dim\n        self.projection = tf.keras.layers.Dense(units=projection_dim)\n        self.position_embedding = tf.keras.layers.Embedding(input_dim=input_dim, output_dim=projection_dim)\n\n    def get_config(self) :\n        config = super(Encoder, self).get_config()\n        config.update({'.input_dim':self.input_dim})\n        return config\n\n    def call(self, inp):\n        positions = tf.range(start=0, limit=self.input_dim, delta=1)\n        encoded = self.projection(inp) + self.position_embedding(positions)\n        return encoded\n\ndef create_vit(inp, input_dim, projection_dim, num_heads, transformer_units,\n               transformer_layers, mlp_head_units, num_outputs) :\n    encoded_inp = Encoder(input_dim, projection_dim)(inp)\n    for _ in range(transformer_layers) :\n        x1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)(encoded_inp)\n        attention_output = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=projection_dim, dropout=0.1)(x1, x1)\n        x2 = tf.keras.layers.Add()([attention_output, encoded_inp])\n        x3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x2)\n        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)\n        encoded_inp = tf.keras.layers.Add()([x3, x2])\n    representation = tf.keras.layers.LayerNormalization(epsilon=1e-6)(encoded_inp)\n    representation = tf.keras.layers.Flatten()(representation)\n    representation = tf.keras.layers.Dropout(0.1)(representation)#0.5\n    features = mlp(representation, hidden_units=mlp_head_units, dropout_rate=0.1)#0.5\n    out = tf.keras.layers.Dense(num_outputs)(features)\n    return out\n\ndef build_model(input_shape, num_outputs) :\n    inputs = tf.keras.layers.Input(shape=input_shape)\n    x = tf.keras.layers.Dense(units=64)(inputs)\n    prediction = create_vit(inp=x, input_dim=91, projection_dim=128, num_heads=2, transformer_units=[128*2, 128],\n                            transformer_layers=8, mlp_head_units=[1028, 256], num_outputs=num_outputs)\n    model = tf.keras.models.Model(inputs=inputs, outputs=prediction)\n    return model\n\ndef threshold_rounder(y_pred, thresholds) :\n    y_pred_round = np.where(y_pred<thresholds[0], 0, np.where(y_pred<thresholds[1], 1, np.where(y_pred<thresholds[2], 2, 3)))\n    return y_pred_round\n\ndef evaluate_predictions(thresholds, y_true, y_pred) :\n    y_pred_round = threshold_rounder(y_pred, thresholds)\n    r_score = cohen_kappa_score(y1=y_true, y2=y_pred_round, weights='quadratic') * (-1)\n    return r_score\n\ndef weighted_predictions(model_weights, y_true, y_pred) :\n    y_pred = np.average(y_pred, axis=0, weights=model_weights)\n    scipy_minimize = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(y_true, y_pred), method='Nelder-Mead')\n    r_score = evaluate_predictions(scipy_minimize.x, y_true, y_pred)\n    return r_score\n\ndef learning(dir, train_data, epochs=50, batch_size=16, n_splits=5, n_outs=1) :\n    df_inp, df_tar, df_id = preprocess(dir, train_data)\n    \n    inp_shape = (df_inp.shape[1], df_inp.shape[-1])\n\n    with tf.device(device_name) :\n        model_reg = build_model(input_shape=inp_shape, num_outputs=n_outs)\n    model_reg.summary()\n\n    model_reg.compile(optimizer=tf.keras.optimizers.AdamW(learning_rate=5e-5), loss=tf.keras.losses.MeanAbsoluteError(),\n                      metrics=[tf.keras.losses.MeanAbsoluteError()])\n    model_reg_weights = model_reg.get_weights()\n\n    lgb_params = {'task':'train', 'boosting_type':'gbdt', 'objective':'regression', 'max_depth':7, 'n_jobs':-1,\n                  'min_child_samples':10, 'num_leaves':int((2**7)*0.7), 'metric':'mae', 'learning_rate':0.01,\n                  'verbosity':-1, 'reg_lambda':9, 'subsample':1.0, 'colsample_bytree':0.7}\n\n    catb_params = {'loss_function':'RMSE', 'num_boost_round':10000, 'depth':7, 'learning_rate':0.03,\n                   'min_child_samples':10, 'verbose':0, 'colsample_bylevel':0.7, 'l2_leaf_reg':9,\n                   'eval_metric':'MAE'}\n    \n    rs = np.random.permutation(np.arange(1, 101))\n    rs_k = 0\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True)\n    vit_fold_valid_pred = []\n    lgb_fold_valid_pred = []\n    catb_fold_valid_pred = []\n    fold_valid_true = []\n    model_b = {}\n    for k, (train_idx, valid_idx) in enumerate(skf.split(X=df_inp, y=df_tar)) :\n        inp = df_inp[train_idx]\n        tar = df_tar[train_idx]\n        valid_inp = df_inp[valid_idx]\n        valid_tar = df_tar[valid_idx]\n        inp_b = np.squeeze(inp)\n        valid_inp_b = np.squeeze(valid_inp)\n\n        model_reg.set_weights(model_reg_weights)\n        checkpoint_filepath = 'fold_'+str(k)+'.weights.h5'\n        checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath, monitor='val_mean_absolute_error', mode='auto',\n                                                                 save_best_only=True, save_weights_only=True,\n                                                                 initial_value_threshold=None)\n        history = model_reg.fit(inp, tar, validation_data=(valid_inp, valid_tar), epochs=epochs, batch_size=batch_size, verbose=0,\n                                callbacks=[checkpoint_callback])\n        #hist = history.history\n        #best_valid_epoch = np.argmin(hist['val_mean_absolute_error'])\n        model_reg.load_weights(checkpoint_filepath)\n        vit_valid_pred = model_reg.predict(valid_inp, verbose=0).flatten().clip(0, 3)\n        #print('fold_'+str(k)+'_epoch_'+str(best_valid_epoch)+'_train_mae:',\n        #      hist['mean_absolute_error'][best_valid_epoch],\n        #      ' valid_mae:', hist['val_mean_absolute_error'][best_valid_epoch])\n        vit_valid_pred_round = threshold_rounder(vit_valid_pred, [0.5, 1.5, 2.5])\n        vit_kappa = cohen_kappa_score(y1=valid_tar, y2=vit_valid_pred_round, weights='quadratic')\n        print('fold_'+str(k)+'_vit_valid_kappa:', vit_kappa)\n\n        lgb_params['random_state'] = rs[rs_k]\n        train_dataset = lgb.Dataset(inp_b, tar)\n        valid_dataset = lgb.Dataset(valid_inp_b, valid_tar, reference=train_dataset)\n        lgb_model = lgb.train(params=lgb_params, train_set=train_dataset, valid_sets=[train_dataset, valid_dataset],\n                              valid_names=['Train', 'Valid'], num_boost_round=10000,\n                              callbacks=[lgb.early_stopping(stopping_rounds=100, verbose=False)])\n        lgb_valid_pred = lgb_model.predict(valid_inp_b).clip(0, 3)\n        lgb_valid_pred_round = threshold_rounder(lgb_valid_pred, [0.5, 1.5, 2.5])\n        lgb_kappa = cohen_kappa_score(y1=valid_tar, y2=lgb_valid_pred_round, weights='quadratic')\n        model_b['fold_'+str(k)+'_lgb'] = lgb_model\n        print('fold_'+str(k)+'_lgb_valid_kappa:', lgb_kappa)\n\n        catb_params['random_state'] = rs[rs_k]\n        train_dataset = Pool(inp_b, tar)\n        valid_dataset = Pool(valid_inp_b, valid_tar) \n        catb_model = CatBoost(params=catb_params)\n        catb_model.fit(train_dataset, verbose=False, eval_set=[valid_dataset], early_stopping_rounds=100)\n        catb_valid_pred = catb_model.predict(valid_inp_b).clip(0, 3)\n        catb_valid_pred_round = threshold_rounder(catb_valid_pred, [0.5, 1.5, 2.5])\n        catb_kappa = cohen_kappa_score(y1=valid_tar, y2=catb_valid_pred_round, weights='quadratic')\n        model_b['fold_'+str(k)+'_catb'] = catb_model\n        print('fold_'+str(k)+'_catb_valid_kappa:', catb_kappa)\n        print('------------------')\n        \n        vit_fold_valid_pred.extend(vit_valid_pred)\n        lgb_fold_valid_pred.extend(lgb_valid_pred)\n        catb_fold_valid_pred.extend(catb_valid_pred)\n        fold_valid_true.extend(valid_tar)\n        \n        rs_k += 1\n\n    print('------------------------------------------')\n    fold_valid_pred = [vit_fold_valid_pred, lgb_fold_valid_pred, catb_fold_valid_pred]\n    scipy_minimize_weights = minimize(weighted_predictions, x0=[1/3, 1/3, 1/3],\n                                                            bounds=[[0, 1], [0, 1], [0, 1]],\n                                                            args=(fold_valid_true, fold_valid_pred),\n                                                            method='Powell')\n    model_weights = scipy_minimize_weights.x / np.sum(scipy_minimize_weights.x)\n    print('model_weights:', model_weights, np.sum(model_weights))\n    ens_valid_pred = np.average(fold_valid_pred, axis=0, weights=model_weights)\n    ens_valid_pred_round = threshold_rounder(ens_valid_pred, [0.5, 1.5, 2.5]) \n    print('ens_valid_sii_kappa:', cohen_kappa_score(y1=fold_valid_true, y2=ens_valid_pred_round, weights='quadratic'))\n    scipy_minimize = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(fold_valid_true, ens_valid_pred), method='Nelder-Mead')\n    ens_valid_pred_round = threshold_rounder(ens_valid_pred, scipy_minimize.x)\n    print('ens_valid_sii_kappa_result:', cohen_kappa_score(y1=fold_valid_true, y2=ens_valid_pred_round, weights='quadratic'))\n\n    return model_reg, model_b, scipy_minimize, model_weights\n\ndef predict(dir, model_reg, model_b, scipy_minimize, model_weights, train_data, test_data, n_splits=5) :\n    df_inp, df_tar, df_id = preprocess(dir, train_data, test_data)\n    df_inp_b = np.squeeze(df_inp)\n\n    fold_pred = []\n    for k in range(n_splits) :\n        checkpoint_filepath = 'fold_'+str(k)+'.weights.h5'\n        model_reg.load_weights(checkpoint_filepath)\n        vit_pred = model_reg.predict(df_inp, verbose=0).flatten().clip(0, 3)\n        lgb_model = model_b['fold_'+str(k)+'_lgb']\n        lgb_pred = lgb_model.predict(df_inp_b).clip(0, 3)\n        catb_model = model_b['fold_'+str(k)+'_catb']\n        catb_pred = catb_model.predict(df_inp_b).clip(0, 3)\n        fold_pred.append(np.average([vit_pred, lgb_pred, catb_pred], axis=0, weights=model_weights))\n    pred_result = np.mean(fold_pred, axis=0)\n    result = threshold_rounder(pred_result, scipy_minimize.x)\n    result = df_id.to_frame().with_columns(sii=result)\n    result.write_csv('submission.csv')\n\nmodel_reg, model_b, scipy_minimize, model_weights = learning(dir, df_train)\npredict(dir, model_reg, model_b, scipy_minimize, model_weights, df_train, df_test)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-27T09:29:39.892978Z","iopub.execute_input":"2024-10-27T09:29:39.893327Z","iopub.status.idle":"2024-10-27T09:47:40.715255Z","shell.execute_reply.started":"2024-10-27T09:29:39.893291Z","shell.execute_reply":"2024-10-27T09:47:40.714109Z"},"trusted":true},"execution_count":null,"outputs":[]}]}