{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7049337,"sourceType":"datasetVersion","datasetId":4056736},{"sourceId":7712331,"sourceType":"datasetVersion","datasetId":4441094}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom random import shuffle\n\nfrom sklearn.base import clone\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nimport tensorflow as tf\nimport tensorflow.keras as keras    \nimport gc\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Activation, Embedding, Flatten, GaussianNoise\nfrom tensorflow.keras.models import Sequential\n\nfrom tensorflow.keras.optimizers.legacy import Adam\nfrom sklearn.decomposition import TruncatedSVD\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-23T10:08:34.493263Z","iopub.execute_input":"2023-11-23T10:08:34.493993Z","iopub.status.idle":"2023-11-23T10:08:34.500572Z","shell.execute_reply.started":"2023-11-23T10:08:34.493964Z","shell.execute_reply":"2023-11-23T10:08:34.499562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reset_tensorflow_keras_backend():\n    tf.keras.backend.clear_session()\n    tf.compat.v1.reset_default_graph()\n    _ = gc.collect()\n\n\ndef load_train_data():\n    train_df = pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\n    train_df = train_df.sample(frac=1.0, random_state=42)\n    return train_df\n\n\ndef mean_rowwise_rmse(y_true, y_pred):\n    rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred), axis=1))\n    mrrmse_score = np.mean(rowwise_rmse)\n    return mrrmse_score\n\ndef abs_error(true, pred):\n    return np.abs(true - pred).mean()\n\n\ndef custom_mean_rowwise_rmse(y_true, y_pred):\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    return mean_rmse","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:08:34.812464Z","iopub.execute_input":"2023-11-23T10:08:34.812797Z","iopub.status.idle":"2023-11-23T10:08:34.823973Z","shell.execute_reply.started":"2023-11-23T10:08:34.812773Z","shell.execute_reply":"2023-11-23T10:08:34.823025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_1(lr, \n            emb_out,\n            n_dim):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        Dense(256),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(0.2),\n        Dense(1024, activation=\"relu\"),\n        BatchNormalization(),\n        Dropout(0.2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\n\ndef model_2(lr, \n            emb_out, \n            dense_1, dense_2, \n            dropout_1, dropout_2,\n            n_dim):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        \n        Dropout(dropout_1), # 256 - 2048\n        Dense(dense_2, activation=\"relu\"),\n        Activation(\"relu\"),\n        BatchNormalization(),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_3(lr, \n            emb_out, \n            dense_1, dense_2, dense_3, dense_4,\n            dropout_1, dropout_2, dropout_3, dropout_4,\n            n_dim):\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 128 - 1024\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        \n        Dense(dense_2), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(dense_3), # 32 - 256\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_3),\n\n        Dense(dense_4), # 16 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_4),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_4(lr, \n            emb_out, \n            dense_1, dense_2, dense_3,\n            dropout_1, dropout_2, dropout_3,\n            n_dim):\n    model = Sequential([\n    Embedding(152, emb_out, input_length=2),\n    Flatten(),\n\n    Dense(dense_1), # 128 - 1024\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_1),\n\n    Dense(dense_2), # 64 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_2),\n        \n    Dense(dense_3), # 32 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_3),\n\n    Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=\"mae\", \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_5(lr, \n            emb_out,\n            n_dim,\n            dropout_1,\n            dropout_2):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        Dense(256),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        Dense(1024),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_6(lr, \n            emb_out, \n            dense_1, dense_2,\n            n_dim,\n            dropout_1,\n            dropout_2):\n    tf.random.set_seed(42)\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        BatchNormalization(),\n        Dense(dense_1), # 64 - 512\n        Activation(\"relu\"),\n        \n        Dropout(dropout_2),\n        Dense(dense_2),\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_7(lr, \n            emb_out, \n            dense_1, dense_2, dense_3, dense_4,\n            dropout_1, dropout_2, dropout_3, dropout_4,\n            n_dim):\n    model = Sequential([\n        Embedding(152, emb_out, input_length=2),\n        Flatten(),\n        \n        Dense(dense_1), # 128 - 1024\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_1),\n        \n        Dense(dense_2), # 64 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_2),\n        \n        Dense(dense_3), # 32 - 256\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_3),\n\n        Dense(dense_4), # 16 - 512\n        BatchNormalization(),\n        Activation(\"relu\"),\n        Dropout(dropout_4),\n        \n        Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n\n\ndef model_8(lr, \n            emb_out, \n            dense_1, dense_2, dense_3,\n            dropout_1, dropout_2, dropout_3,\n            n_dim):\n    model = Sequential([\n    Embedding(152, emb_out, input_length=2),\n    Flatten(),\n\n    Dense(dense_1), # 128 - 1024\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_1),\n\n    Dense(dense_2), # 64 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_2),\n        \n    Dense(dense_3), # 32 - 512\n    BatchNormalization(),\n    Activation(\"relu\"),\n    Dropout(dropout_3),\n\n    Dense(n_dim, activation= \"linear\")\n    ])\n\n    model.compile(loss=custom_mean_rowwise_rmse, \n                    optimizer=Adam(learning_rate=lr),\n                 metrics=[custom_mean_rowwise_rmse])\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:08:35.204179Z","iopub.execute_input":"2023-11-23T10:08:35.204525Z","iopub.status.idle":"2023-11-23T10:08:35.232122Z","shell.execute_reply.started":"2023-11-23T10:08:35.204495Z","shell.execute_reply":"2023-11-23T10:08:35.231186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load_train_data().reset_index(drop=True)\noriginal_x = train_df[['cell_type', 'sm_name']].values\noriginal_y = train_df.loc[:, 'A1BG':].values\nle = LabelEncoder()\nle.fit(original_x.flat)\nnew_names = le.transform(original_x.flat).reshape(-1, 2)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:08:35.644567Z","iopub.execute_input":"2023-11-23T10:08:35.644936Z","iopub.status.idle":"2023-11-23T10:08:36.756736Z","shell.execute_reply.started":"2023-11-23T10:08:35.644906Z","shell.execute_reply":"2023-11-23T10:08:36.755899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_params_to_training_model(model_params):\n    model_params = model_params['params']\n    training_keys = ['epochs', 'bs']\n    training_params = {k: model_params[k] for k in training_keys}\n    model_params = {k: model_params[k] for k in model_params.keys() if k not in training_keys}\n    return model_params, training_params","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:08:36.812277Z","iopub.execute_input":"2023-11-23T10:08:36.812557Z","iopub.status.idle":"2023-11-23T10:08:36.820061Z","shell.execute_reply.started":"2023-11-23T10:08:36.812533Z","shell.execute_reply":"2023-11-23T10:08:36.819049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_model_1 =     {\n        \"params\": {\n            \"epochs\": 114,\n            \"bs\": 128,\n            \"lr\": 0.008457844054540857,\n            \"emb_out\": 22,\n            \"n_dim\": 50\n        },\n        \"value\": 0.9060678655727635\n    }\n\nparams_model_2 =     {\n        \"params\": {\n            \"epochs\": 136,\n            \"bs\": 64,\n            \"lr\": 0.007787474024659863,\n            \"emb_out\": 10,\n            \"dense_1\": 384,\n            \"dense_2\": 1280,\n            \"dropout_1\": 0.4643149193312417,\n            \"dropout_2\": 0.10101884612160547,\n            \"n_dim\": 60\n        },\n        \"value\": 0.9070240468092804\n    }\n\nparams_model_3 =     {\n        \"params\": {\n            \"epochs\": 157,\n            \"bs\": 64,\n            \"lr\": 0.004311857150745656,\n            \"emb_out\": 62,\n            \"dense_1\": 560,\n            \"dense_2\": 480,\n            \"dense_3\": 248,\n            \"dense_4\": 224,\n            \"dropout_1\": 0.4359908049836846,\n            \"dropout_2\": 0.34432694543970555,\n            \"dropout_3\": 0.01112409967333259,\n            \"dropout_4\": 0.23133616975077548,\n            \"n_dim\": 119\n        },\n        \"value\": 0.9171315640806535\n    }\n\nparams_model_4 =     {\n        \"params\": {\n            \"epochs\": 147,\n            \"bs\": 64,\n            \"lr\": 0.005948541271442179,\n            \"emb_out\": 46,\n            \"dense_1\": 872,\n            \"dense_2\": 264,\n            \"dense_3\": 256,\n            \"dropout_1\": 0.17543603718794346,\n            \"dropout_2\": 0.3587657616370447,\n            \"dropout_3\": 0.12077512068514727,\n            \"n_dim\": 213\n        },\n        \"value\": 0.9228638968500431\n    }\n\nparams_model_5 =     {\n        \"params\": {\n            \"epochs\": 122,\n            \"bs\": 32,\n            \"lr\": 0.004429076555977599,\n            \"emb_out\": 32,\n            \"n_dim\": 71,\n            \"dropout_1\": 0.40604535344002984,\n            \"dropout_2\": 0.178189970426619\n        },\n        \"value\": 0.9083640103276015\n    }\n\nparams_model_6 =     {\n        \"params\": {\n            \"epochs\": 112,\n            \"bs\": 128,\n            \"lr\": 0.009773732221901085,\n            \"emb_out\": 60,\n            \"dense_1\": 436,\n            \"dense_2\": 416,\n            \"n_dim\": 126,\n            \"dropout_1\": 0.4024659444883379,\n            \"dropout_2\": 0.2573940194596736\n        },\n        \"value\": 0.8909352668212382\n    }\n\nparams_model_7 =     {\n        \"params\": {\n            \"epochs\": 141,\n            \"bs\": 128,\n            \"lr\": 0.005530331519967936,\n            \"emb_out\": 48,\n            \"dense_1\": 712,\n            \"dense_2\": 400,\n            \"dense_3\": 232,\n            \"dense_4\": 216,\n            \"dropout_1\": 0.4903998136177629,\n            \"dropout_2\": 0.032371643764537134,\n            \"dropout_3\": 0.11138300987168903,\n            \"dropout_4\": 0.019885384663655765,\n            \"n_dim\": 100\n        },\n        \"value\": 0.8978272722102707\n    }\n\nparams_model_8 =     {\n        \"params\": {\n            \"epochs\": 143,\n            \"bs\": 192,\n            \"lr\": 0.00971858172843266,\n            \"emb_out\": 48,\n            \"dense_1\": 312,\n            \"dense_2\": 344,\n            \"dense_3\": 248,\n            \"dropout_1\": 0.10974777738609129,\n            \"dropout_2\": 0.10106027333885811,\n            \"dropout_3\": 0.09775833250663657,\n            \"n_dim\": 100\n        },\n        \"value\": 0.8885448573595669\n    }","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:08:43.620421Z","iopub.execute_input":"2023-11-23T10:08:43.620749Z","iopub.status.idle":"2023-11-23T10:08:43.631538Z","shell.execute_reply.started":"2023-11-23T10:08:43.620724Z","shell.execute_reply":"2023-11-23T10:08:43.630613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w1 = [0.15224443321212433, 0.7152220796128623, 0.7547606691460997, 0.05786285275052854, 0.9602177109190158, 0.4968056740470425, 0.9881673272809887]","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:10:00.772324Z","iopub.execute_input":"2023-11-23T10:10:00.773043Z","iopub.status.idle":"2023-11-23T10:10:00.777027Z","shell.execute_reply.started":"2023-11-23T10:10:00.773013Z","shell.execute_reply":"2023-11-23T10:10:00.77614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_and_predict_embedding_nn(x, y, test_x, model_constructor, best_params):\n    model_params, training_params = split_params_to_training_model(best_params)\n    n_dim = model_params['n_dim']\n    d = TruncatedSVD(n_dim)\n    y = d.fit_transform(y)\n    model = model_constructor(**model_params)\n    model.fit(x, y, epochs=training_params['epochs'], \n                    batch_size=training_params['bs'], \n                    verbose=0,\n                    shuffle=True)\n    return d.inverse_transform(model.predict(test_x, batch_size=1))","metadata":{"execution":{"iopub.status.busy":"2023-11-23T10:10:02.25257Z","iopub.execute_input":"2023-11-23T10:10:02.253342Z","iopub.status.idle":"2023-11-23T10:10:02.259507Z","shell.execute_reply.started":"2023-11-23T10:10:02.253309Z","shell.execute_reply":"2023-11-23T10:10:02.258559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reps = 10\n\ntest_df = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\n\ndef predict(test_df):\n    x_test = le.transform(test_df[['cell_type', 'sm_name']].values.flat).reshape(-1, 2)\n    \n    preds_model_1 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_1, params_model_1) for i in range(reps)]\n    preds_model_2 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_2, params_model_2) for i in range(reps)]\n    preds_model_3 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_3, params_model_3) for i in range(reps)]\n    preds_model_5 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_5, params_model_5) for i in range(reps)]\n    preds_model_6 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_6, params_model_6) for i in range(reps)]\n    preds_model_7 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_7, params_model_7) for i in range(reps)]\n    preds_model_8 = [fit_and_predict_embedding_nn(new_names, original_y, x_test, model_8, params_model_8) for i in range(reps)]\n    \n    pred1 = np.median(preds_model_1, axis=0)\n    pred2 = np.median(preds_model_2, axis=0)\n    pred3 = np.median(preds_model_3, axis=0)\n    pred5 = np.median(preds_model_5, axis=0)\n    pred6 = np.median(preds_model_6, axis=0)\n    pred7 = np.median(preds_model_7, axis=0)\n    pred8 = np.median(preds_model_8, axis=0)\n    \n    pred = (w1[0] * pred1 + w1[1] * pred2 + w1[2] * pred3 + w1[3] * pred5 + w1[4] * pred6 + w1[5] * pred7 + w1[6] * pred8) / sum(w1)\n    \n    return pred\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-23T08:28:25.92883Z","iopub.execute_input":"2023-11-23T08:28:25.929557Z","iopub.status.idle":"2023-11-23T08:28:29.908725Z","shell.execute_reply.started":"2023-11-23T08:28:25.929524Z","shell.execute_reply":"2023-11-23T08:28:29.907704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mins = original_y.min(axis=0)\nmaxs = original_y.max(axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predict(test_df)\nclipped_pred = np.clip(pred, mins, maxs)\n\ndf = pd.DataFrame(clipped_pred, columns = sample_submission.columns[1:])\ndf['id'] = range(len(df))\ndf = df.loc[:, sample_submission.columns]","metadata":{"execution":{"iopub.status.busy":"2023-11-23T08:28:29.91056Z","iopub.execute_input":"2023-11-23T08:28:29.910941Z","iopub.status.idle":"2023-11-23T08:29:35.900436Z","shell.execute_reply.started":"2023-11-23T08:28:29.910907Z","shell.execute_reply":"2023-11-23T08:29:35.899555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T08:29:35.901852Z","iopub.execute_input":"2023-11-23T08:29:35.902211Z","iopub.status.idle":"2023-11-23T08:29:47.038437Z","shell.execute_reply.started":"2023-11-23T08:29:35.902177Z","shell.execute_reply":"2023-11-23T08:29:47.037606Z"},"trusted":true},"execution_count":null,"outputs":[]}]}