{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc, pickle, datetime, scipy.sparse\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom colorama import Fore, Back, Style\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import TruncatedSVD,PCA\nfrom sklearn.metrics import mean_squared_error\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\n\nimport scipy.sparse\n\nDATA_DIR = \"../input/open-problems-multimodal\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")\n\nVERBOSE = 0","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-06T11:32:02.687686Z","iopub.execute_input":"2022-12-06T11:32:02.688116Z","iopub.status.idle":"2022-12-06T11:32:03.360904Z","shell.execute_reply.started":"2022-12-06T11:32:02.688009Z","shell.execute_reply":"2022-12-06T11:32:03.359971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ------ CITEseq MODEL ---------","metadata":{}},{"cell_type":"code","source":"metadata_df = pd.read_csv(FP_CELL_METADATA, index_col = 'cell_id')\nmetadata_df = metadata_df[metadata_df.technology == \"citeseq\"]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:35:04.231347Z","iopub.execute_input":"2022-12-06T11:35:04.232040Z","iopub.status.idle":"2022-12-06T11:35:04.514611Z","shell.execute_reply.started":"2022-12-06T11:35:04.232001Z","shell.execute_reply":"2022-12-06T11:35:04.513648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_index = np.load(\"/kaggle/input/multimodal-single-cell-as-sparse-matrix/train_cite_inputs_idxcol.npz\",allow_pickle=True)[\"index\"]\nmeta = metadata_df.reindex(train_index)\nmeta.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:35:41.894167Z","iopub.execute_input":"2022-12-06T11:35:41.895287Z","iopub.status.idle":"2022-12-06T11:35:41.958112Z","shell.execute_reply.started":"2022-12-06T11:35:41.895238Z","shell.execute_reply":"2022-12-06T11:35:41.956999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_index_test = np.load(\"/kaggle/input/multimodal-single-cell-as-sparse-matrix/test_cite_inputs_idxcol.npz\",allow_pickle=True)[\"index\"]\ncell_index_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:36:08.595716Z","iopub.execute_input":"2022-12-06T11:36:08.596065Z","iopub.status.idle":"2022-12-06T11:36:08.631918Z","shell.execute_reply.started":"2022-12-06T11:36:08.596033Z","shell.execute_reply":"2022-12-06T11:36:08.630887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target normalization","metadata":{}},{"cell_type":"code","source":"X = np.load(\"../input/cite-final/new_cite_train_final.npz\")[\"arr_0\"]\nXt = np.load(\"../input/cite-final/new_cite_test_final.npz\")[\"arr_0\"]\nX.shape,Xt.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:34:11.316608Z","iopub.execute_input":"2022-12-06T11:34:11.316958Z","iopub.status.idle":"2022-12-06T11:34:16.001137Z","shell.execute_reply.started":"2022-12-06T11:34:11.316927Z","shell.execute_reply":"2022-12-06T11:34:15.999950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = pd.read_hdf(FP_CITE_TRAIN_TARGETS)\nY = Y.values\nY -= Y.mean(axis=1).reshape(-1, 1)\nY /= Y.std(axis=1).reshape(-1, 1)\nY.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:34:17.319166Z","iopub.execute_input":"2022-12-06T11:34:17.319517Z","iopub.status.idle":"2022-12-06T11:34:18.012055Z","shell.execute_reply.started":"2022-12-06T11:34:17.319488Z","shell.execute_reply":"2022-12-06T11:34:18.011107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tensorflow Keras librairies","metadata":{}},{"cell_type":"code","source":"import math\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping\nfrom tensorflow.keras.layers import Dense, Input, Concatenate, Dropout, BatchNormalization","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:34:33.984709Z","iopub.execute_input":"2022-12-06T11:34:33.985121Z","iopub.status.idle":"2022-12-06T11:34:38.841172Z","shell.execute_reply.started":"2022-12-06T11:34:33.985064Z","shell.execute_reply":"2022-12-06T11:34:38.840166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric and loss function","metadata":{}},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n\ndef negative_correlation_loss(y_true, y_pred):\n    my = K.mean(tf.convert_to_tensor(y_pred), axis=1)\n    my = tf.tile(tf.expand_dims(my, axis=1), (1, y_true.shape[1]))\n    ym = y_pred - my\n    r_num = K.sum(tf.multiply(y_true, ym), axis=1)\n    r_den = tf.sqrt(K.sum(K.square(ym), axis=1) * float(y_true.shape[-1]))\n    r = tf.reduce_mean(r_num / r_den)\n    return - r","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:34:38.843085Z","iopub.execute_input":"2022-12-06T11:34:38.843773Z","iopub.status.idle":"2022-12-06T11:34:38.855863Z","shell.execute_reply.started":"2022-12-06T11:34:38.843734Z","shell.execute_reply":"2022-12-06T11:34:38.853939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and parameters","metadata":{}},{"cell_type":"code","source":"LR_START = 0.01\nBATCH_SIZE = 512\n\ndef create_model():\n    \n    reg1 = 9.613e-06\n    reg2 = 1e-07\n    REG1 = tf.keras.regularizers.l2(reg1)\n    REG2 = tf.keras.regularizers.l2(reg2)\n    DROP = 0.1\n\n    activation = 'selu'\n    inputs = Input(shape =(X.shape[1],))\n\n    x0 = Dense(512, \n              kernel_regularizer = REG1,\n              activation = activation,\n             )(inputs)\n    x0 = Dropout(DROP)(x0)\n    \n    \n    x1 = Dense(512, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x0)\n    x1 = Dropout(DROP)(x1)\n    \n    \n    x2 = Dense(512, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x1) \n    x2= Dropout(DROP)(x2)\n    \n    x3 = Dense(256, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x2) \n    x3= Dropout(DROP)(x3)\n    \n    x4 = Dense(Y.shape[1],\n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x3)\n    x4 = Dropout(DROP)(x4)\n\n         \n    x = Concatenate()([\n \n                x2, \n                x3,\n                x4\n                ])\n    \n#     x = Dense(256, \n#                 kernel_regularizer = REG2,\n#                 activation='linear',\n#                 )(x)\n    \n    x = Dense(Y.shape[1], \n            kernel_regularizer = REG2,\n            activation='linear',\n            )(x)\n    \n    \n    model = Model(inputs, x)\n    \n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:34:38.858765Z","iopub.execute_input":"2022-12-06T11:34:38.859794Z","iopub.status.idle":"2022-12-06T11:34:38.873733Z","shell.execute_reply.started":"2022-12-06T11:34:38.859747Z","shell.execute_reply":"2022-12-06T11:34:38.872720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:36:14.459688Z","iopub.execute_input":"2022-12-06T11:36:14.460043Z","iopub.status.idle":"2022-12-06T11:36:14.466197Z","shell.execute_reply.started":"2022-12-06T11:36:14.460010Z","shell.execute_reply":"2022-12-06T11:36:14.465159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:36:15.945858Z","iopub.execute_input":"2022-12-06T11:36:15.946241Z","iopub.status.idle":"2022-12-06T11:36:15.952766Z","shell.execute_reply.started":"2022-12-06T11:36:15.946206Z","shell.execute_reply":"2022-12-06T11:36:15.951548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tqdm.notebook import tqdm\n# meta[\"id\"] = [i for i in range(meta.shape[0])]\n# people_list = [32606,13176,31800]\n# day_list = [2,3,4]\n# FOLDS_LIST = []\n# num_fold = 3\n\n# for val_people in [32606,13176,31800]:\n#     train_people = [i for i in people_list if i != val_people]\n#     for val_day in tqdm(day_list):\n#         train_day = [i for i in day_list if i != val_day]\n#         train_idx = meta[meta.day.isin(train_day)][meta.donor.isin(train_people)].id.to_list()\n#         val_idx = meta[meta.day == val_day].id.to_list()\n        \n\n#         one_type = [\n#             train_idx,val_idx\n#         ]\n#         FOLDS_LIST.append(one_type)\n# len(FOLDS_LIST)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T19:42:50.241503Z","iopub.execute_input":"2022-11-15T19:42:50.241858Z","iopub.status.idle":"2022-11-15T19:42:50.645227Z","shell.execute_reply.started":"2022-11-15T19:42:50.241829Z","shell.execute_reply":"2022-11-15T19:42:50.644308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"%%time\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nEPOCHS = 300 \nN_SPLITS = 3\n\npred_train = np.zeros((Y.shape[0],Y.shape[1]))\n\nnp.random.seed(1)\ntf.random.set_seed(1)\nscore_list = []\nkf = GroupKFold(n_splits=N_SPLITS)\nscore_list = []\n\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(X, groups=meta.donor)):\n    start_time = datetime.datetime.now()\n    model = None\n    gc.collect()\n    \n    X_tr = X[idx_tr]\n    y_tr = Y[idx_tr]\n    X_va = X[idx_va]\n    y_va = Y[idx_va]\n\n    lr = ReduceLROnPlateau(\n                    monitor = \"val_loss\",\n                    factor = 0.9, \n                    patience = 4, \n                    verbose = VERBOSE)\n\n    es = EarlyStopping(\n                    monitor = \"val_loss\",\n                    patience = 40, \n                    verbose = VERBOSE,\n                    mode = \"min\", \n                    restore_best_weights = True)\n\n    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n                    filepath = './citeseq',\n                    save_weights_only = True,\n                    monitor = 'val_loss',\n                    mode = 'min',\n                    save_best_only = True)\n\n    callbacks = [\n                    lr, \n                    es, \n                    model_checkpoint_callback\n                    ]\n    \n    model = create_model()\n    \n    model.compile(\n                optimizer = tf.keras.optimizers.Adam(learning_rate=LR_START),\n                metrics = [negative_correlation_loss],\n                loss = negative_correlation_loss\n                 )\n    # Training\n    model.fit(\n                X_tr,\n                y_tr, \n                validation_data=(\n                                X_va,\n                                y_va), \n                epochs = EPOCHS,\n                verbose = VERBOSE,\n                batch_size = BATCH_SIZE,\n                shuffle = True,\n                callbacks = callbacks)\n\n    del X_tr, y_tr \n    gc.collect()\n    \n    model.load_weights('./citeseq')\n    model.save(f\"./submissions/model_{fold}\")\n    print('model saved')\n    \n    #  Model validation\n    y_va_pred = model.predict(X_va)\n    corrscore = correlation_score(y_va, y_va_pred)\n    pred_train[idx_va] = y_va_pred\n    \n    print(f\"Fold {fold}, correlation =  {corrscore:.5f}\")\n    del X_va, y_va, y_va_pred\n    gc.collect()\n    score_list.append(corrscore)\n\n# Show overall score\nprint(f\"{Fore.GREEN}{Style.BRIGHT}Mean corr = {np.array(score_list).mean():.5f}{Style.RESET_ALL}\")\nscore_total = correlation_score(Y, pred_train)\nprint(f\"{Fore.BLUE}{Style.BRIGHT}Oof corr   = {score_total:.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:37:23.864012Z","iopub.execute_input":"2022-12-06T11:37:23.864409Z","iopub.status.idle":"2022-12-06T11:43:59.008472Z","shell.execute_reply.started":"2022-12-06T11:37:23.864375Z","shell.execute_reply":"2022-12-06T11:43:59.007145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CITEseq Test prediction","metadata":{}},{"cell_type":"code","source":"weights = [0.89399,0.89825,0.89547]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:51:11.247351Z","iopub.execute_input":"2022-12-06T11:51:11.247742Z","iopub.status.idle":"2022-12-06T11:51:11.254401Z","shell.execute_reply.started":"2022-12-06T11:51:11.247706Z","shell.execute_reply":"2022-12-06T11:51:11.253286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def std(x):\n    return (x - np.mean(x,axis=1).reshape(-1,1)) / np.std(x,axis=1).reshape(-1,1)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:51:12.425359Z","iopub.execute_input":"2022-12-06T11:51:12.425942Z","iopub.status.idle":"2022-12-06T11:51:12.432476Z","shell.execute_reply.started":"2022-12-06T11:51:12.425906Z","shell.execute_reply":"2022-12-06T11:51:12.431318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.zeros((len(Xt), 140), dtype=np.float32)\nfor fold in range(N_SPLITS):\n    print(f\"Predicting with fold {fold}\")\n    model = load_model(f\"./submissions/model_{fold}\",\n                       custom_objects={'negative_correlation_loss': negative_correlation_loss})\n    test_pred += std(model.predict(Xt))*weights[fold]\n\n# from Juan Smith Perera to complete with the Multiome part :\nsubmission = pd.read_csv('/kaggle/input/4th-solution-ensemble/submission.zip',index_col='row_id', squeeze=True)\nsubmission.iloc[:len(test_pred.ravel())] = test_pred.ravel()\nassert not submission.isna().any()\n\nsubmission.to_csv('submission.csv')\ndisplay(submission)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:02:06.074834Z","iopub.execute_input":"2022-12-06T12:02:06.075235Z","iopub.status.idle":"2022-12-06T12:05:52.217994Z","shell.execute_reply.started":"2022-12-06T12:02:06.075198Z","shell.execute_reply":"2022-12-06T12:05:52.217048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:05:52.219954Z","iopub.execute_input":"2022-12-06T12:05:52.220386Z","iopub.status.idle":"2022-12-06T12:05:52.227629Z","shell.execute_reply.started":"2022-12-06T12:05:52.220343Z","shell.execute_reply":"2022-12-06T12:05:52.226396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}