{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7049337,"sourceType":"datasetVersion","datasetId":4056736},{"sourceId":7631230,"sourceType":"datasetVersion","datasetId":4441094}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom random import shuffle\n\nfrom sklearn.base import clone\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nimport tensorflow as tf\nimport tensorflow.keras as keras    \nimport gc\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Activation, Embedding, Flatten, GaussianNoise\nfrom tensorflow.keras.models import Sequential\n\nfrom tensorflow.keras.optimizers.legacy import Adam\nfrom sklearn.decomposition import TruncatedSVD\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-30T07:11:29.485817Z","iopub.execute_input":"2023-11-30T07:11:29.486135Z","iopub.status.idle":"2023-11-30T07:11:39.146417Z","shell.execute_reply.started":"2023-11-30T07:11:29.48611Z","shell.execute_reply":"2023-11-30T07:11:39.145257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reset_tensorflow_keras_backend():\n    tf.keras.backend.clear_session()\n    tf.compat.v1.reset_default_graph()\n    _ = gc.collect()\n\n\ndef load_train_data():\n    train_df = pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\n    train_df = train_df.sample(frac=1.0, random_state=42)\n    return train_df\n\n\ndef mean_rowwise_rmse(y_true, y_pred):\n    rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred), axis=1))\n    mrrmse_score = np.mean(rowwise_rmse)\n    return mrrmse_score\n\ndef abs_error(true, pred):\n    return np.abs(true - pred).mean()\n\n\ndef split_over_all_test(df):\n    valid_folds = []\n    sm_names_B_M = pd.unique(df[df['cell_type'].isin(['B cells', 'Myeloid cells'])]['sm_name'])\n    for cell_type in ['NK cells', 'T cells CD4+', 'T cells CD8+', 'T regulatory cells']:\n        sub_df = df[(df['cell_type'] == cell_type) & (~df['sm_name'].isin(sm_names_B_M))]\n        valid_folds.append(sub_df.index.tolist())\n    folds = []\n    all_train = set(range(len(df)))\n    for fold in valid_folds:\n        fold_set = set(fold)\n        train_rest = list(all_train - fold_set)\n        shuffle(train_rest)\n        shuffle(fold)\n        folds.append((train_rest, fold))\n    return folds\n\n\ndef custom_mean_rowwise_rmse(y_true, y_pred):\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    return mean_rmse","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:11:39.148252Z","iopub.execute_input":"2023-11-30T07:11:39.148923Z","iopub.status.idle":"2023-11-30T07:11:39.162783Z","shell.execute_reply.started":"2023-11-30T07:11:39.148888Z","shell.execute_reply":"2023-11-30T07:11:39.161885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_1(lr, \n            emb_out,\n            n_dim):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        Dense(256),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(0.2),\n        Dense(1024, activation=\"relu\"),\n        BatchNormalization(),\n        Dropout(0.2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\n\ndef model_2(lr, \n            emb_out, \n            dense_1, dense_2, \n            dropout_1, dropout_2,\n            n_dim):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        \n        Dropout(dropout_1), # 256 - 2048\n        Dense(dense_2, activation=\"relu\"),\n        Activation(\"relu\"),\n        BatchNormalization(),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_3(lr, \n            emb_out, \n            dense_1, dense_2, dense_3, dense_4,\n            dropout_1, dropout_2, dropout_3, dropout_4,\n            n_dim):\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 128 - 1024\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        \n        Dense(dense_2), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(dense_3), # 32 - 256\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_3),\n\n        Dense(dense_4), # 16 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_4),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_4(lr, \n            emb_out, \n            dense_1, dense_2, dense_3,\n            dropout_1, dropout_2, dropout_3,\n            n_dim):\n    model = Sequential([\n    Embedding(152, emb_out, input_length=2),\n    Flatten(),\n\n    Dense(dense_1), # 128 - 1024\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_1),\n\n    Dense(dense_2), # 64 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_2),\n        \n    Dense(dense_3), # 32 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_3),\n\n    Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_5(lr, \n            emb_out,\n            n_dim,\n            dropout_1,\n            dropout_2):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        Dense(256),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        Dense(1024),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_6(lr, \n            emb_out, \n            dense_1, dense_2,\n            n_dim,\n            dropout_1,\n            dropout_2):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        BatchNormalization(),\n        Dense(dense_1), # 64 - 512\n        Activation(\"relu\"),\n        \n        Dropout(dropout_2),\n        Dense(dense_2),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_7(lr, \n            emb_out, \n            dense_1, dense_2, dense_3, dense_4,\n            dropout_1, dropout_2, dropout_3, dropout_4,\n            n_dim):\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 128 - 1024\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        \n        Dense(dense_2), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(dense_3), # 32 - 256\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_3),\n\n        Dense(dense_4), # 16 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_4),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_8(lr, \n            emb_out, \n            dense_1, dense_2, dense_3,\n            dropout_1, dropout_2, dropout_3,\n            n_dim):\n    model = Sequential([\n    Embedding(152, emb_out, input_length=2),\n    Flatten(),\n\n    Dense(dense_1), # 128 - 1024\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_1),\n\n    Dense(dense_2), # 64 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_2),\n        \n    Dense(dense_3), # 32 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_3),\n\n    Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:11:39.165705Z","iopub.execute_input":"2023-11-30T07:11:39.166299Z","iopub.status.idle":"2023-11-30T07:11:39.202691Z","shell.execute_reply.started":"2023-11-30T07:11:39.166266Z","shell.execute_reply":"2023-11-30T07:11:39.201875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudolabel = pd.read_csv('/kaggle/input/pseudolabel/submission(14).csv')\ntest_df = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\npseudolabel = pd.concat([test_df[['cell_type', 'sm_name']], pseudolabel.loc[:, 'A1BG':]], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:11:39.204513Z","iopub.execute_input":"2023-11-30T07:11:39.204868Z","iopub.status.idle":"2023-11-30T07:11:45.119485Z","shell.execute_reply.started":"2023-11-30T07:11:39.204816Z","shell.execute_reply":"2023-11-30T07:11:45.118684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load_train_data().reset_index(drop=True)\ncolumns = ['cell_type', 'sm_name'] + train_df.loc[:, 'A1BG':].columns.tolist()\ntrain_df = train_df.loc[:, columns]","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:12:07.7525Z","iopub.execute_input":"2023-11-30T07:12:07.753282Z","iopub.status.idle":"2023-11-30T07:12:09.013327Z","shell.execute_reply.started":"2023-11-30T07:12:07.75325Z","shell.execute_reply":"2023-11-30T07:12:09.01225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v = train_df.loc[:, 'A1BG':]","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:13:56.976685Z","iopub.execute_input":"2023-11-30T07:13:56.977694Z","iopub.status.idle":"2023-11-30T07:13:57.013084Z","shell.execute_reply.started":"2023-11-30T07:13:56.977649Z","shell.execute_reply":"2023-11-30T07:13:57.011768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_df, pseudolabel]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:16:50.232098Z","iopub.execute_input":"2023-11-30T07:16:50.232457Z","iopub.status.idle":"2023-11-30T07:16:50.332566Z","shell.execute_reply.started":"2023-11-30T07:16:50.23243Z","shell.execute_reply":"2023-11-30T07:16:50.331622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_over_all_test(df, df_orig):\n    valid_folds = []\n    sm_names_B_M = pd.unique(df_orig[df_orig['cell_type'].isin(['B cells', 'Myeloid cells'])]['sm_name'])\n    for cell_type in ['NK cells', 'T cells CD4+', 'T cells CD8+', 'T regulatory cells']:\n        sub_df = df[(df['cell_type'] == cell_type) & (~df['sm_name'].isin(sm_names_B_M))]\n        valid_folds.append(sub_df.index.tolist())\n    folds = []\n    all_train = set(range(len(df)))\n    for fold in valid_folds:\n        fold_set = set(fold)\n        train_rest = list(all_train - fold_set)\n        shuffle(train_rest)\n        shuffle(fold)\n        folds.append((train_rest, fold))\n    return folds\n\nover_all_test = split_over_all_test(train_df, load_train_data().reset_index(drop=True))","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:16:51.200738Z","iopub.execute_input":"2023-11-30T07:16:51.201172Z","iopub.status.idle":"2023-11-30T07:16:52.402322Z","shell.execute_reply.started":"2023-11-30T07:16:51.201139Z","shell.execute_reply":"2023-11-30T07:16:52.401405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_x = train_df[['cell_type', 'sm_name']].values\noriginal_y = train_df.loc[:, 'A1BG':].values\nle = LabelEncoder()\nle.fit(original_x.flat)\nnew_names = le.transform(original_x.flat).reshape(-1, 2)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:16:57.600845Z","iopub.execute_input":"2023-11-30T07:16:57.602141Z","iopub.status.idle":"2023-11-30T07:16:57.651817Z","shell.execute_reply.started":"2023-11-30T07:16:57.602093Z","shell.execute_reply":"2023-11-30T07:16:57.650902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_params_to_training_model(model_params):\n    model_params = model_params['params']\n    training_keys = ['epochs', 'bs']\n    training_params = {k: model_params[k] for k in training_keys}\n    model_params = {k: model_params[k] for k in model_params.keys() if k not in training_keys}\n    return model_params, training_params","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:09.048593Z","iopub.execute_input":"2023-11-30T07:17:09.049028Z","iopub.status.idle":"2023-11-30T07:17:09.054688Z","shell.execute_reply.started":"2023-11-30T07:17:09.048997Z","shell.execute_reply":"2023-11-30T07:17:09.053742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 200","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:17.824738Z","iopub.execute_input":"2023-11-30T07:17:17.825149Z","iopub.status.idle":"2023-11-30T07:17:17.829537Z","shell.execute_reply.started":"2023-11-30T07:17:17.825118Z","shell.execute_reply":"2023-11-30T07:17:17.828619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_model_1a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.008457844054540857,\n            \"emb_out\": 22,\n            \"n_dim\": 50\n        },\n        \"value\": 0.9060678655727635\n    }\n\n\n####\n\nparams_model_2a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 64,\n            \"lr\": 0.007787474024659863,\n            \"emb_out\": 10,\n            \"dense_1\": 384,\n            \"dense_2\": 1280,\n            \"dropout_1\": 0.4643149193312417,\n            \"dropout_2\": 0.10101884612160547,\n            \"n_dim\": 60\n        },\n        \"value\": 0.9070240468092804\n    }\n\nparams_model_2b = {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 192,\n            \"lr\": 0.00830447680398929,\n            \"emb_out\": 32,\n            \"dense_1\": 284,\n            \"dense_2\": 1424,\n            \"dropout_1\": 0.27860934847913565,\n            \"dropout_2\": 0.04217965884576308,\n            \"n_dim\": 96\n        },\n        \"value\": 1.0158228546558994\n    }\n\n\nparams_model_2c =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 64,\n            \"lr\": 0.006661059864181284,\n            \"emb_out\": 22,\n            \"dense_1\": 232,\n            \"dense_2\": 1184,\n            \"dropout_1\": 0.46230673331531297,\n            \"dropout_2\": 0.24430331733550426,\n            \"n_dim\": 61\n        },\n        \"value\": 0.5378740563213127\n    }\n\n#####\n\nparams_model_3a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 64,\n            \"lr\": 0.004311857150745656,\n            \"emb_out\": 62,\n            \"dense_1\": 560,\n            \"dense_2\": 480,\n            \"dense_3\": 248,\n            \"dense_4\": 224,\n            \"dropout_1\": 0.4359908049836846,\n            \"dropout_2\": 0.34432694543970555,\n            \"dropout_3\": 0.01112409967333259,\n            \"dropout_4\": 0.23133616975077548,\n            \"n_dim\": 119\n        },\n        \"value\": 0.9171315640806535\n    }\n\n\nparams_model_3b =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 64,\n            \"lr\": 0.007915642160705914,\n            \"emb_out\": 22,\n            \"dense_1\": 504,\n            \"dense_2\": 136,\n            \"dense_3\": 232,\n            \"dense_4\": 512,\n            \"dropout_1\": 0.0072011388198520605,\n            \"dropout_2\": 0.07781770809801486,\n            \"dropout_3\": 0.3482776196327668,\n            \"dropout_4\": 0.4010684312497648,\n            \"n_dim\": 55\n        },\n        \"value\": 1.0557613871962215\n    }\n\n\n####\n\nparams_model_4a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 64,\n            \"lr\": 0.005948541271442179,\n            \"emb_out\": 46,\n            \"dense_1\": 872,\n            \"dense_2\": 264,\n            \"dense_3\": 256,\n            \"dropout_1\": 0.17543603718794346,\n            \"dropout_2\": 0.3587657616370447,\n            \"dropout_3\": 0.12077512068514727,\n            \"n_dim\": 213\n        },\n        \"value\": 0.9228638968500431\n    }\n\n\nparams_model_4b =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.006444109866334638,\n            \"emb_out\": 62,\n            \"dense_1\": 552,\n            \"dense_2\": 480,\n            \"dense_3\": 216,\n            \"dropout_1\": 0.323390730123547,\n            \"dropout_2\": 0.15142047240687942,\n            \"dropout_3\": 0.034625791669279364,\n            \"n_dim\": 104\n        },\n        \"value\": 0.8462075069648056\n    }\n\n\n#####\n\n\nparams_model_5a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 32,\n            \"lr\": 0.005430251128204367,\n            \"emb_out\": 56,\n            \"n_dim\": 108,\n            \"dropout_1\": 0.02868537022302934,\n            \"dropout_2\": 0.35808251111776157\n        },\n        \"value\": 0.835768615578779\n    }\n\nparams_model_5b = {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 192,\n            \"lr\": 0.00528860262509972,\n            \"emb_out\": 60,\n            \"n_dim\": 78,\n            \"dropout_1\": 0.2978319229273037,\n            \"dropout_2\": 0.3236224036130246\n        },\n        \"value\": 1.0125358317737336\n    }\n\n\nparams_model_5c =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 32,\n            \"lr\": 0.004429076555977599,\n            \"emb_out\": 32,\n            \"n_dim\": 71,\n            \"dropout_1\": 0.40604535344002984,\n            \"dropout_2\": 0.178189970426619\n        },\n        \"value\": 0.9083640103276015\n    }\n\n\n####\n\nparams_model_6a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.0030468340279031702,\n            \"emb_out\": 62,\n            \"dense_1\": 396,\n            \"dense_2\": 912,\n            \"n_dim\": 144,\n            \"dropout_1\": 0.2643057707162437,\n            \"dropout_2\": 0.1738090239074675\n        },\n        \"value\": 0.8388383786625531\n    }\n\n\nparams_model_6b =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.009773732221901085,\n            \"emb_out\": 60,\n            \"dense_1\": 436,\n            \"dense_2\": 416,\n            \"n_dim\": 126,\n            \"dropout_1\": 0.4024659444883379,\n            \"dropout_2\": 0.2573940194596736\n        },\n        \"value\": 0.8909352668212382\n    }\n\n\nparams_model_6c = {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 160,\n            \"lr\": 0.005742157072582258,\n            \"emb_out\": 56,\n            \"dense_1\": 504,\n            \"dense_2\": 928,\n            \"n_dim\": 134,\n            \"dropout_1\": 0.26460638891781607,\n            \"dropout_2\": 0.243272371789527\n        },\n        \"value\": 0.9921304350469378\n    }\n\n\n####\n\nparams_model_7a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.0026256302897014814,\n            \"emb_out\": 62,\n            \"dense_1\": 824,\n            \"dense_2\": 184,\n            \"dense_3\": 208,\n            \"dense_4\": 472,\n            \"dropout_1\": 0.04406850232282358,\n            \"dropout_2\": 0.051203939042409885,\n            \"dropout_3\": 0.05926676325711479,\n            \"dropout_4\": 0.08819762697219703,\n            \"n_dim\": 167\n        },\n        \"value\": 0.8347070421058967\n    }\n\nparams_model_7b =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 128,\n            \"lr\": 0.005530331519967936,\n            \"emb_out\": 48,\n            \"dense_1\": 712,\n            \"dense_2\": 400,\n            \"dense_3\": 232,\n            \"dense_4\": 216,\n            \"dropout_1\": 0.4903998136177629,\n            \"dropout_2\": 0.032371643764537134,\n            \"dropout_3\": 0.11138300987168903,\n            \"dropout_4\": 0.019885384663655765,\n            \"n_dim\": 100\n        },\n        \"value\": 0.8978272722102707\n    }\n\n\n####\n\n\nparams_model_8a =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 192,\n            \"lr\": 0.00971858172843266,\n            \"emb_out\": 48,\n            \"dense_1\": 312,\n            \"dense_2\": 344,\n            \"dense_3\": 248,\n            \"dropout_1\": 0.10974777738609129,\n            \"dropout_2\": 0.10106027333885811,\n            \"dropout_3\": 0.09775833250663657,\n            \"n_dim\": 100\n        },\n        \"value\": 0.8885448573595669\n    }\n\n\nparams_model_8b =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 192,\n            \"lr\": 0.008078165473745607,\n            \"emb_out\": 16,\n            \"dense_1\": 1016,\n            \"dense_2\": 392,\n            \"dense_3\": 176,\n            \"dropout_1\": 0.21410737149365255,\n            \"dropout_2\": 0.40541433561062473,\n            \"dropout_3\": 0.10476819447155189,\n            \"n_dim\": 72\n        },\n        \"value\": 0.767725144592772\n    }\n\nparams_model_8c =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 160,\n            \"lr\": 0.005427125417330768,\n            \"emb_out\": 36,\n            \"dense_1\": 760,\n            \"dense_2\": 416,\n            \"dense_3\": 240,\n            \"dropout_1\": 0.16485069317527304,\n            \"dropout_2\": 0.014216669745902685,\n            \"dropout_3\": 0.05820818430142793,\n            \"n_dim\": 128\n        },\n        \"value\": 0.9778518605441292\n    }\n\nparams_model_8d =     {\n        \"params\": {\n            \"epochs\": epochs,\n            \"bs\": 224,\n            \"lr\": 0.0077454113093514835,\n            \"emb_out\": 38,\n            \"dense_1\": 856,\n            \"dense_2\": 352,\n            \"dense_3\": 112,\n            \"dropout_1\": 0.058963634508929205,\n            \"dropout_2\": 0.10928657766717247,\n            \"dropout_3\": 0.06218368685386452,\n            \"n_dim\": 246\n        },\n        \"value\": 0.837621399656469\n    }","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:18.731539Z","iopub.execute_input":"2023-11-30T07:17:18.731915Z","iopub.status.idle":"2023-11-30T07:17:18.763804Z","shell.execute_reply.started":"2023-11-30T07:17:18.731886Z","shell.execute_reply":"2023-11-30T07:17:18.762809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [model_1] + [model_2] * 3 + [model_3] * 2 + [model_4] * 2 + [model_5] * 3 + [model_6] * 3 + [model_7] * 2 + [model_8] * 4\nparams = [params_model_1a,\n         params_model_2a, params_model_2b, params_model_2c,\n         params_model_3a, params_model_3b,\n         params_model_4a, params_model_4b,\n         params_model_5a, params_model_5b, params_model_5c,\n         params_model_6a, params_model_6b, params_model_6c,\n         params_model_7a, params_model_7b,\n         params_model_8a, params_model_8b, params_model_8c, params_model_8d]","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:19.899288Z","iopub.execute_input":"2023-11-30T07:17:19.89971Z","iopub.status.idle":"2023-11-30T07:17:19.907716Z","shell.execute_reply.started":"2023-11-30T07:17:19.899679Z","shell.execute_reply":"2023-11-30T07:17:19.906864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = {'w_0': 0.039503611057797205, 'w_1': 0.22778329445024798, 'w_2': 0.6470451053292054, 'w_3': 0.14998376919294348,\n           'w_4': 0.2417986187654036, 'w_5': 0.018231625997453538, 'w_6': 0.0670756518443389, 'w_7': 0.45476025839087764, \n           'w_8': 0.8953651930541969, 'w_9': 0.2341721081339142, 'w_10': 0.3780523499542964, 'w_11': 0.3428728268932388,\n           'w_12': 0.58025233963141, 'w_13': 0.815817039069762, 'w_14': 0.4865022144120589, 'w_15': 0.01707600530474928, \n           'w_16': 0.26623418232070073, 'w_17': 0.20727099624448486, 'w_18': 0.47400340366957744, 'w_19': 0.6758531294442794}\n\nweights = list(weights.values())","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:24.544447Z","iopub.execute_input":"2023-11-30T07:17:24.544818Z","iopub.status.idle":"2023-11-30T07:17:24.551543Z","shell.execute_reply.started":"2023-11-30T07:17:24.544786Z","shell.execute_reply":"2023-11-30T07:17:24.550388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_and_predict_embedding_nn(x, y, test_x, model_constructor, best_params):\n    model_params, training_params = split_params_to_training_model(best_params)\n    n_dim = model_params['n_dim']\n    d = TruncatedSVD(n_dim)\n    y = d.fit_transform(y)\n    model = model_constructor(**model_params)\n    model.fit(x, y, epochs=training_params['epochs'], \n                    batch_size=training_params['bs'], \n                    verbose=0,\n                    shuffle=True)\n    return d.inverse_transform(model.predict(test_x, batch_size=1))","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:17:25.448382Z","iopub.execute_input":"2023-11-30T07:17:25.448766Z","iopub.status.idle":"2023-11-30T07:17:25.455181Z","shell.execute_reply.started":"2023-11-30T07:17:25.448738Z","shell.execute_reply":"2023-11-30T07:17:25.454158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reps = 10\n\ntest_df = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\n\ndef predict(test_df, models, params, weights):\n    x_test = le.transform(test_df[['cell_type', 'sm_name']].values.flat).reshape(-1, 2)\n    \n    preds = []\n    for model, param in zip(models, params):\n        temp_pred = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model, param) for i in range(reps)]\n        temp_pred = np.median(temp_pred, axis=0)\n        preds.append(temp_pred)\n    \n    pred = np.sum([w * p for w, p in zip(weights, preds)], axis=0) / sum(weights)    \n    return pred\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:18:05.279836Z","iopub.execute_input":"2023-11-30T07:18:05.280546Z","iopub.status.idle":"2023-11-30T07:18:09.401186Z","shell.execute_reply.started":"2023-11-30T07:18:05.280513Z","shell.execute_reply":"2023-11-30T07:18:09.400325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predict(test_df, models, params, weights)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T07:18:14.447659Z","iopub.execute_input":"2023-11-30T07:18:14.448043Z","iopub.status.idle":"2023-11-30T08:02:12.312741Z","shell.execute_reply.started":"2023-11-30T07:18:14.448012Z","shell.execute_reply":"2023-11-30T08:02:12.311883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mins = v.min(axis=0).values\nmaxs = v.max(axis=0).values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clipped_pred = np.clip(pred, mins, maxs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(clipped_pred, columns = sample_submission.columns[1:])\ndf['id'] = range(len(df))\ndf = df.loc[:, sample_submission.columns]","metadata":{"execution":{"iopub.status.busy":"2023-11-30T08:06:31.51429Z","iopub.execute_input":"2023-11-30T08:06:31.515269Z","iopub.status.idle":"2023-11-30T08:06:31.544845Z","shell.execute_reply.started":"2023-11-30T08:06:31.515223Z","shell.execute_reply":"2023-11-30T08:06:31.543745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}