{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":38128,"databundleVersionId":4230952,"sourceType":"competition"},{"sourceId":4521534,"sourceType":"datasetVersion","datasetId":2638820}],"dockerImageVersionId":30302,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook performs the ablation study for rank 2 solution. This submission was actually the most successful in solving the Cite-seq task. We will explore what is behind this success and which parts of the model contribute to such an impressive performance.\n\nUnfortunately, Kaggle fails to run the entire notebook due to out of RAM error. You can run different parts, uncommenting cells with experiments without a problem. Here, we'll only run default models for an example","metadata":{}},{"cell_type":"markdown","source":"# Load feature and target","metadata":{}},{"cell_type":"code","source":"%%time\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\n\nimport gc\n\nfeature_path = '/kaggle/input/single-cell-features/'\n\ntrain_df = pd.read_feather(feature_path+'train_cite_inputs_id.feather')\ntest_df = pd.read_feather(feature_path+'test_cite_inputs_id.feather')\n\ntrain_cite_X = np.load(feature_path+'train_cite_X.npy')\ntest_cite_X = np.load(feature_path+'test_cite_X.npy')\n\ntrain_cite_y = np.load(feature_path+'train_cite_targets.npy')  ","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.342098Z","iopub.execute_input":"2025-07-28T12:03:09.342426Z","iopub.status.idle":"2025-07-28T12:03:09.720144Z","shell.execute_reply.started":"2025-07-28T12:03:09.342401Z","shell.execute_reply":"2025-07-28T12:03:09.719140Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCHS = 100\nSEED = 666\nN_FOLD = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-28T12:03:09.722116Z","iopub.execute_input":"2025-07-28T12:03:09.722848Z","iopub.status.idle":"2025-07-28T12:03:09.727113Z","shell.execute_reply.started":"2025-07-28T12:03:09.722809Z","shell.execute_reply":"2025-07-28T12:03:09.726111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs(\"models/\", exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.728322Z","iopub.execute_input":"2025-07-28T12:03:09.728581Z","iopub.status.idle":"2025-07-28T12:03:09.742918Z","shell.execute_reply.started":"2025-07-28T12:03:09.728557Z","shell.execute_reply":"2025-07-28T12:03:09.742237Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to the code in [this notebook](https://github.com/senkin13/kaggle/blob/master/Open-Problems-Multimodal-Single-Cell-Integration-2nd-Place-Solution/senkin13/cite_nn.ipynb), CITE dataset contains several inputs:\n- SVD-transformed and CLR-preprocessed expression data – 200 features\n- SVD of normalized data corrected for each day – 100 features\n- Important features from raw data (highly correlated with target) – 245 features. This number is not clear from the code but can be calculated based on the others\n- PCA-transformed raw data – 64 features\n- LGBM predictions from 4 models with different inputs – 4 * 100 features","metadata":{}},{"cell_type":"code","source":"n_features_per_dataset = [200, 100, 245, 64, 100, 100, 100, 100]  # 245 = 1009 - rest\nnp.cumsum(n_features_per_dataset)","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.745079Z","iopub.execute_input":"2025-07-28T12:03:09.745327Z","iopub.status.idle":"2025-07-28T12:03:09.757273Z","shell.execute_reply.started":"2025-07-28T12:03:09.745305Z","shell.execute_reply":"2025-07-28T12:03:09.756530Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_features = train_cite_X.shape[1]\nn_features","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.758147Z","iopub.execute_input":"2025-07-28T12:03:09.758360Z","iopub.status.idle":"2025-07-28T12:03:09.774480Z","shell.execute_reply.started":"2025-07-28T12:03:09.758340Z","shell.execute_reply":"2025-07-28T12:03:09.773729Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_idxs = np.arange(n_features)\n\ndata_subsets = {\n    \"no_preprocessed_expression_data\": ~np.isin(all_idxs, np.arange(199)),  # equiv to slice(201, n_features) here\n    \"no_day_corrected_data\": ~np.isin(all_idxs, np.arange(200, 299)),  # Drop idxs from 200 to 300\n    \"no_raw_data_with_selected_features\": ~np.isin(all_idxs, np.arange(300, 544)),\n    \"no_raw_data_pca\": ~np.isin(all_idxs, np.arange(545, 608)),\n    \"no_lgbm_1\": ~np.isin(all_idxs, np.arange(609, 708)),\n    \"no_lgbm_2\": ~np.isin(all_idxs, np.arange(709, 808)),\n    \"no_lgbm_3\": ~np.isin(all_idxs, np.arange(809, 908)),\n    \"no_lgbm_4\": ~np.isin(all_idxs, np.arange(909, 1008)),\n    \"no_lgbm\": ~np.isin(all_idxs, np.arange(609, 1008)),\n    \"no_lgbm_and_raw\": ~np.isin(all_idxs, np.arange(300, 1008)),\n}\n","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.775359Z","iopub.execute_input":"2025-07-28T12:03:09.775614Z","iopub.status.idle":"2025-07-28T12:03:09.841800Z","shell.execute_reply.started":"2025-07-28T12:03:09.775581Z","shell.execute_reply":"2025-07-28T12:03:09.841136Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nsns.heatmap(pd.DataFrame(data_subsets).T)","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:09.842787Z","iopub.execute_input":"2025-07-28T12:03:09.843050Z","iopub.status.idle":"2025-07-28T12:03:10.733553Z","shell.execute_reply.started":"2025-07-28T12:03:09.843028Z","shell.execute_reply":"2025-07-28T12:03:10.732621Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 20))\n\nax = sns.heatmap(train_cite_X, cmap=\"coolwarm\")\n\n# Highlight where the datasets start\nfor idx in np.cumsum(n_features_per_dataset)[:-1]:\n    ax.axvline(x=idx - 1, color='black', linewidth=1)","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:03:10.734659Z","iopub.execute_input":"2025-07-28T12:03:10.735015Z","iopub.status.idle":"2025-07-28T12:04:33.080548Z","shell.execute_reply.started":"2025-07-28T12:03:10.734989Z","shell.execute_reply":"2025-07-28T12:04:33.079311Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.shape, test_df.shape, train_cite_X.shape, test_cite_X.shape, train_cite_y.shape","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:04:33.081833Z","iopub.execute_input":"2025-07-28T12:04:33.082184Z","iopub.status.idle":"2025-07-28T12:04:33.088997Z","shell.execute_reply.started":"2025-07-28T12:04:33.082153Z","shell.execute_reply":"2025-07-28T12:04:33.087914Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedKFold,KFold,GroupKFold\n\ndef correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n\ndef zscore(x):\n    x_zscore = []\n    for i in range(x.shape[0]):\n        x_row = x[i]\n        x_row = (x_row - np.mean(x_row)) / np.std(x_row)\n        x_zscore.append(x_row)\n    x_std = np.array(x_zscore)    \n    return x_std\n\ndef cosine_similarity_loss(y_true, y_pred):\n    x = y_true\n    y = y_pred\n    mx = tf.reduce_mean(x, axis=1, keepdims=True)\n    my = tf.reduce_mean(y, axis=1, keepdims=True)\n    xm, ym = x - mx, y - my\n    t1_norm = tf.math.l2_normalize(xm, axis = 1)\n    t2_norm = tf.math.l2_normalize(ym, axis = 1)\n    cosine = tf.keras.losses.CosineSimilarity(axis = 1)(t1_norm, t2_norm)\n    return cosine\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, train_X, train_y, list_IDs, shuffle, batch_size, labels, ): \n        self.train_X = train_X\n        self.train_y = train_y\n        self.list_IDs = list_IDs        \n        self.shuffle = shuffle\n        self.batch_size = batch_size\n        self.labels = labels\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = len(self.list_IDs) // self.batch_size\n        return ct\n    \n    def __getitem__(self, idx):\n        'Generate one batch of data'\n        indexes = self.list_IDs[idx*self.batch_size:(idx+1)*self.batch_size]\n    \n        # Find list of IDs\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        X, y = self.__data_generation(list_IDs_temp)\n\n        if self.labels: return X, y\n        else: return X\n \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.list_IDs) )\n        if self.shuffle: \n            np.random.shuffle(self.indexes)\n            \n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples'    \n        X = self.train_X[list_IDs_temp]\n        y = self.train_y[list_IDs_temp]        \n        return X, y\n    \ndef nn_kfold(train_df, train_cite_X, train_cite_y, test_df, test_cite_X, network, folds, model_name, **kwargs):\n    print(\"Additional parameters:\", kwargs)\n    \n    oof_preds = np.zeros((train_df.shape[0],140))\n    sub_preds = np.zeros((test_df.shape[0],140))\n    cv_corr = []\n    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df,)):          \n        print (n_fold)\n        train_x = train_cite_X[train_idx]\n        valid_x = train_cite_X[valid_idx]\n        train_y = train_cite_y[train_idx]\n        valid_y = train_cite_y[valid_idx]\n\n        train_x_index = train_df.iloc[train_idx].reset_index(drop=True).index\n        valid_x_index = train_df.iloc[valid_idx].reset_index(drop=True).index\n        \n        model = network(train_cite_X.shape[1], **kwargs)\n        filepath = model_name+'_'+str(n_fold)+'.h5'\n        es = tf.keras.callbacks.EarlyStopping(patience=10, mode='min', verbose=0) \n        checkpoint = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath=filepath, save_best_only=True,save_weights_only=True,mode='min') \n        reduce_lr_loss = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=LR_FACTOR, patience=6, verbose=0)\n    \n        train_dataset = DataGenerator(\n            train_x,\n            train_y,\n            list_IDs=train_x_index, \n            shuffle=True, \n            batch_size=BATCH_SIZE, \n            labels=True,\n        )\n        \n        valid_dataset = DataGenerator(\n            valid_x,\n            valid_y,\n            list_IDs=valid_x_index, \n            shuffle=False, \n            batch_size=BATCH_SIZE, \n            labels=True,\n        )\n    \n        hist = model.fit(train_dataset,\n                        validation_data=valid_dataset,  \n                        epochs=EPOCHS, \n                        callbacks=[checkpoint,es,reduce_lr_loss],\n                        workers=4,\n                        verbose=0) \n        print(\"HISTORY:\", hist.history)\n    \n        model.load_weights(filepath)\n        \n        oof_preds[valid_idx] = model.predict(valid_x, \n                                batch_size=BATCH_SIZE,\n                                verbose=1)\n        \n        oof_corr = correlation_score(valid_y,  oof_preds[valid_idx])\n        cv_corr.append(oof_corr)\n        print (cv_corr)       \n        \n        sub_preds += model.predict(test_cite_X, \n                                batch_size=BATCH_SIZE,\n                                verbose=1) / folds.n_splits \n            \n        del model\n        gc.collect()\n        tf.keras.backend.clear_session()    \n    cv = correlation_score(train_cite_y, oof_preds)\n    print ('Overall:',cv)           \n    return oof_preds,sub_preds    \n\n\ndef train_model(\n    train_df,\n    train_cite_X,\n    train_cite_y,\n    test_df,\n    test_cite_X,\n    network,\n    folds,\n    model_name,\n#     sub,\n#     multi_preds,\n    **kwargs\n):\n    oof_preds_mse, sub_preds_mse = nn_kfold(\n        train_df,\n        train_cite_X,\n        train_cite_y,\n        test_df,\n        test_cite_X,\n        network,\n        folds,\n        model_name,\n        **kwargs\n    )\n    \n    oof_preds = zscore(oof_preds_mse)\n    cv = correlation_score(train_cite_y,  oof_preds)\n    print ('CV score:', cv)   \n\n    sub_preds = zscore(sub_preds_mse) \n    \n#     sub_preds = sub_preds.reshape(-1)\n\n#     # Merge with multiome, save the result\n#     sub['target'] = np.concatenate([sub_preds,multi_preds])\n#     sub.to_csv(f'ablation_{data_subset}_submission.csv',index=False)\n    \n    return sub_preds\n","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:04:33.091822Z","iopub.execute_input":"2025-07-28T12:04:33.092092Z","iopub.status.idle":"2025-07-28T12:04:33.115695Z","shell.execute_reply.started":"2025-07-28T12:04:33.092070Z","shell.execute_reply":"2025-07-28T12:04:33.114770Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-28T12:04:33.116736Z","iopub.execute_input":"2025-07-28T12:04:33.116996Z","iopub.status.idle":"2025-07-28T12:04:33.132418Z","shell.execute_reply.started":"2025-07-28T12:04:33.116963Z","shell.execute_reply":"2025-07-28T12:04:33.131605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model1 - cosine similarity loss","metadata":{}},{"cell_type":"markdown","source":"We won't use this model for ablation studies. It is a bit more complex that the MSE one due to concatenation part, but the performance of 2 models is very comparable. So, we'll evaluate only the default models","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef cite_cos_sim_model(len_num):\n    \n    #######################  svd  #######################   \n    input_num = tf.keras.Input(shape=(len_num))     \n    x = input_num\n    x0 =  tf.keras.layers.Reshape((1,x.shape[1]))(x)\n    x0 = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(1800, activation='elu', kernel_initializer='Identity',return_sequences=False))(x0)\n    x1 = tf.keras.layers.GaussianDropout(0.2)(x0)         \n    x2 = tf.keras.layers.Dense(1800,activation ='elu',kernel_initializer='Identity',)(x1) \n    x3 = tf.keras.layers.GaussianDropout(0.2)(x2) \n    x4 = tf.keras.layers.Dense(1800,activation ='elu',kernel_initializer='Identity',)(x3) \n    x5 = tf.keras.layers.GaussianDropout(0.2)(x4)         \n    x = tf.keras.layers.Concatenate()([\n                       x1,x3,x5\n                      ])\n    output = tf.keras.layers.Dense(140, activation='linear')(x) \n    model = tf.keras.models.Model(input_num, output)\n    lr=0.001\n    adam = tf.keras.optimizers.Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, epsilon=None, )\n    model.compile(loss=cosine_similarity_loss, optimizer=adam,)\n    model.summary()\n    return model\n\n\nBATCH_SIZE = 620\nLR_FACTOR = 0.05","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:04:33.133358Z","iopub.execute_input":"2025-07-28T12:04:33.133603Z","iopub.status.idle":"2025-07-28T12:04:33.149110Z","shell.execute_reply.started":"2025-07-28T12:04:33.133582Z","shell.execute_reply":"2025-07-28T12:04:33.148109Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Evaluate the default cos model","metadata":{}},{"cell_type":"code","source":"y_test_pred_cos_default = train_model(\n    train_df=train_df,\n    train_cite_X=train_cite_X,\n    train_cite_y=train_cite_y,\n    test_df=test_df,\n    test_cite_X=test_cite_X,\n    network=cite_cos_sim_model,\n    folds=folds,\n    model_name=f'cite_cos_sim_model_default',\n#         sub=sub,\n#         multi_preds=multi_preds,\n)\nnp.save(f'default_cos_cite_prediction.npy', y_test_pred_cos_default)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-28T12:04:33.150071Z","iopub.execute_input":"2025-07-28T12:04:33.150321Z","iopub.status.idle":"2025-07-28T12:06:07.081720Z","shell.execute_reply.started":"2025-07-28T12:04:33.150299Z","shell.execute_reply":"2025-07-28T12:06:07.080668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del y_test_pred_cos_default\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model2 - mse loss","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef cite_mse_model(\n    len_num, \n    dropout_strategy='gaussian', \n    activation='swish', \n    width=1500, \n    num_blocks=3, \n    use_recurrent='gru'\n):\n    \"\"\"\n    Creates a neural network model for predicting Cite-seq data with configurable ablations.\n\n    Parameters\n    ----------\n    len_num : int\n        Input shape length.\n    dropout_strategy : str, optional\n        Dropout strategy to use, one of 'gaussian', 'normal', or 'no'. Default is 'gaussian'.\n    activation : str, optional\n        Activation function to use in Dense layers, one of 'relu' or 'swish'. Default is 'swish'.\n    width : int, optional\n        Number of neurons in each Dense layer. Default is 1500.\n    num_blocks : int, optional\n        Number of Dense-Dropout blocks. Default is 3.\n    use_recurrent : str, optional\n        Type of recurrent layer to use, one of 'gru', 'lstm', 'linear', or 'none'. Default is 'gru'.\n\n    Returns\n    -------\n    tf.keras.models.Model\n        Compiled Keras model.\n\"\"\"\n\n    \n    input_num = tf.keras.Input(shape=(len_num))     \n\n    x = input_num\n    \n    # Define dropout function based on strategy\n    def get_dropout_layer(rate):\n        if dropout_strategy == 'gaussian':\n            return tf.keras.layers.GaussianDropout(rate)\n        elif dropout_strategy == 'normal':\n            return tf.keras.layers.Dropout(rate)\n        else:\n            return lambda x: x\n    \n    # Add Dense and Dropout blocks\n    for _ in range(num_blocks):\n        x = tf.keras.layers.Dense(width, activation=activation)(x)\n        x = get_dropout_layer(0.1)(x)\n    \n    # Recurrent layer or reshape based on use_recurrent\n    if use_recurrent in ['gru', 'lstm']:\n        x = tf.keras.layers.Reshape((1, x.shape[1]))(x)\n        if use_recurrent == 'gru':\n            x = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(700, activation=activation, return_sequences=False))(x)\n        elif use_recurrent == 'lstm':\n            x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(700, activation=activation, return_sequences=False))(x)\n    elif use_recurrent == 'linear':\n        x = tf.keras.layers.Dense(700, activation=activation)(x)\n    \n    # Final dropout layer if not 'none'\n    if use_recurrent != 'none':\n        x = get_dropout_layer(0.1)(x)\n    \n    output = tf.keras.layers.Dense(140, activation='linear')(x) \n\n    model = tf.keras.models.Model(input_num, output)\n    \n    lr = 0.0005\n    weight_decay = 0.0001\n    \n    opt = tfa.optimizers.AdamW(\n        learning_rate=lr, weight_decay=weight_decay\n    )    \n\n    model.compile(loss=tf.keras.losses.MeanSquaredError(), optimizer=opt)\n    model.summary()\n    return model\n\nBATCH_SIZE = 600\nLR_FACTOR = 0.1\n\n# zscore for target\ntrain_cite_y = zscore(train_cite_y)","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:06:07.085317Z","iopub.execute_input":"2025-07-28T12:06:07.085630Z","iopub.status.idle":"2025-07-28T12:06:09.662666Z","shell.execute_reply.started":"2025-07-28T12:06:07.085606Z","shell.execute_reply":"2025-07-28T12:06:09.661488Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Evaluate the default MSE model","metadata":{}},{"cell_type":"code","source":"y_test_pred_mse_default = train_model(\n    train_df=train_df,\n    train_cite_X=train_cite_X,\n    train_cite_y=train_cite_y,\n    test_df=test_df,\n    test_cite_X=test_cite_X,\n    network=cite_mse_model,\n    folds=folds,\n    model_name=f'cite_mse_model_default',\n#         sub=sub,\n#         multi_preds=multi_preds,\n)\nnp.save(f'default_mse_cite_prediction.npy', y_test_pred_mse_default)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-28T12:06:09.665298Z","iopub.execute_input":"2025-07-28T12:06:09.665556Z","iopub.status.idle":"2025-07-28T12:07:03.727795Z","shell.execute_reply.started":"2025-07-28T12:06:09.665532Z","shell.execute_reply":"2025-07-28T12:07:03.726995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del y_test_pred_mse_default\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Evaluate MSE model trained on different data parts","metadata":{}},{"cell_type":"code","source":"# Uncomment to run data ablation experiments\n\n# %%time\n\n# for data_subset, mask in data_subsets.items():\n#     print(\"Working with data subset\", data_subset)\n#     print(\"Features used:\", train_cite_X[:, mask].shape[1])\n#     y_test_pred = train_model(\n#         train_df=train_df,\n#         train_cite_X=train_cite_X[:, mask],\n#         train_cite_y=train_cite_y,\n#         test_df=test_df,\n#         test_cite_X=test_cite_X[:, mask],\n#         network=cite_mse_model,\n#         folds=folds,\n#         model_name=f'cite_mse_model_{data_subset}',\n# #         sub=sub,\n# #         multi_preds=multi_preds\n#     )\n#     np.save(f'{data_subset}_cite_prediction.npy', y_test_pred)\n\n#     del y_test_pred\n#     gc.collect()","metadata":{"execution":{"iopub.status.busy":"2025-07-28T12:07:03.730686Z","iopub.execute_input":"2025-07-28T12:07:03.730929Z","execution_failed":"2025-07-28T12:13:12.517Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Ablate some parts of the MSE model","metadata":{}},{"cell_type":"code","source":"# Uncomment to run model ablation experiments\n\n# %%time\n\n# ablations = {\n#     \"normal_dropout\": {\"dropout_strategy\": \"normal\"},\n#     \"no_dropout\": {\"dropout_strategy\": \"no\"},\n#     \"relu_activation\": {\"activation\": \"relu\"},\n#     \"fc_width_1000\": {\"width\": 1000},\n#     \"fc_width_500\": {\"width\": 500},\n#     \"one_fc_block\": {\"num_blocks\": 1},\n#     \"two_fc_blocks\": {\"num_blocks\": 2},\n#     \"lstm_recurrent\": {\"use_recurrent\": \"lstm\"},\n#     \"linear_recurrent\": {\"use_recurrent\": \"linear\"},\n#     \"no_recurrent\": {\"use_recurrent\": \"none\"}\n# }\n\n# for ablation_name, network_params in ablations.items():\n#     print(\"Performing ablation experiment\", ablation_name)\n#     y_test_pred = train_model(\n#         train_df=train_df,\n#         train_cite_X=train_cite_X,\n#         train_cite_y=train_cite_y,\n#         test_df=test_df,\n#         test_cite_X=test_cite_X,\n#         network=cite_mse_model,\n#         folds=folds,\n#         model_name=f'cite_mse_model_{ablation_name}',\n# #         sub=sub,\n# #         multi_preds=multi_preds,\n#         **network_params\n#     )\n#     np.save(f'{ablation_name}_cite_prediction.npy', y_test_pred)\n\n#     del y_test_pred\n#     gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-28T12:13:12.524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"multi_preds = np.load(feature_path + 'senkin_multi_predictions.npy')\nsub = pd.read_csv('../input/open-problems-multimodal/sample_submission.csv')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-28T12:13:12.524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"experiments = (\n    # default models\n    \"default_cos\",\n    \"default_mse\"\n    # data_ablations\n    # \"no_preprocessed_expression_data\",\n    # \"no_day_corrected_data\",\n    # \"no_raw_data_with_selected_features\",\n    # \"no_raw_data_pca\",\n    # # lgbm_ablations\n    # \"no_lgbm_1\",\n    # \"no_lgbm_2\",\n    # \"no_lgbm_3\",\n    # \"no_lgbm_4\",\n    # \"no_lgbm\",\n    # \"no_lgbm_and_raw\",\n    # # model_ablations\n    # \"relu_activation\",\n    # \"fc_width_1000\",\n    # \"fc_width_500\",\n    # \"one_fc_block\",\n    # \"two_fc_blocks\",\n    # # gru_ablations\n    # \"normal_dropout\",\n    # \"no_dropout\",\n    # \"lstm_recurrent\",\n    # \"linear_recurrent\",\n    # \"no_recurrent\"\n)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-28T12:13:12.524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for experiment in experiments:\n    sub_preds = np.load(f\"{experiment}_cite_prediction.npy\")\n\n    sub_preds = sub_preds.reshape(-1)\n\n    # Merge with multiome, save the result\n    sub['target'] = np.concatenate([sub_preds, multi_preds])\n    sub.to_csv(f'{experiment}_submission.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-28T12:13:12.525Z"}},"outputs":[],"execution_count":null}]}