{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## KERAS for Multiome\n\nIdeas and code from :\n\nhttps://www.kaggle.com/code/xiafire/lb-t15-msci-multiome-catboostregressor\n\nhttps://www.kaggle.com/code/ambrosm/msci-citeseq-keras-quickstart\n\ndataset from :\n\nhttps://www.kaggle.com/datasets/fabiencrom/multimodal-single-cell-as-sparse-matrix\n\n\nMy appologies for the Kagglers not mentioned.","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:48:20.894278Z","iopub.execute_input":"2022-10-11T02:48:20.895149Z","iopub.status.idle":"2022-10-11T02:48:20.930016Z","shell.execute_reply.started":"2022-10-11T02:48:20.895012Z","shell.execute_reply":"2022-10-11T02:48:20.927666Z"}}},{"cell_type":"code","source":"! pip install tables","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:57:23.665235Z","iopub.execute_input":"2022-10-11T02:57:23.665669Z","iopub.status.idle":"2022-10-11T02:57:36.487161Z","shell.execute_reply.started":"2022-10-11T02:57:23.665583Z","shell.execute_reply":"2022-10-11T02:57:36.48613Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Requirement already satisfied: tables in /opt/conda/lib/python3.7/site-packages (3.7.0)\nRequirement already satisfied: numexpr>=2.6.2 in /opt/conda/lib/python3.7/site-packages (from tables) (2.8.3)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tables) (21.3)\nRequirement already satisfied: numpy>=1.19.0 in /opt/conda/lib/python3.7/site-packages (from tables) (1.21.6)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->tables) (3.0.9)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import gc,os,pickle\n\nimport pandas as pd\nimport numpy as np\n\nimport glob\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import mean_squared_error\n\nimport time\nimport psutil\nimport tables\n\nimport scipy.sparse","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-10-11T02:57:36.48978Z","iopub.execute_input":"2022-10-11T02:57:36.490492Z","iopub.status.idle":"2022-10-11T02:57:37.38619Z","shell.execute_reply.started":"2022-10-11T02:57:36.490438Z","shell.execute_reply":"2022-10-11T02:57:37.385239Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"## Target already normalized and reduced\nDue to memory limitation, the target has been normalized and reduced by TruncatedSVD (n_comp = 128) in another Notebook and available in the dataset '../input/targets-multiome-sparse-scaled/'","metadata":{}},{"cell_type":"code","source":"%%time\nwith open('../input/targets-multiome-sparse-scaled/Y.pkl','rb') as f: Y = pickle.load(f)\nY.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:57:37.387454Z","iopub.execute_input":"2022-10-11T02:57:37.387786Z","iopub.status.idle":"2022-10-11T02:57:38.202527Z","shell.execute_reply.started":"2022-10-11T02:57:37.387756Z","shell.execute_reply":"2022-10-11T02:57:38.20132Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"CPU times: user 18.8 ms, sys: 96.3 ms, total: 115 ms\nWall time: 805 ms\n","output_type":"stream"},{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"(105942, 128)"},"metadata":{}}]},{"cell_type":"markdown","source":"## Target dimension reduction ","metadata":{}},{"cell_type":"code","source":"with open('../input/targets-multiome-sparse-scaled/pca_target.pkl','rb') as f: pca_target = pickle.load(f)\npca_target","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:57:38.205045Z","iopub.execute_input":"2022-10-11T02:57:38.205423Z","iopub.status.idle":"2022-10-11T02:57:38.335699Z","shell.execute_reply.started":"2022-10-11T02:57:38.20539Z","shell.execute_reply":"2022-10-11T02:57:38.334614Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/sklearn/base.py:338: UserWarning: Trying to unpickle estimator TruncatedSVD from version 1.1.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\nhttps://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n  UserWarning,\n","output_type":"stream"},{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"TruncatedSVD(n_components=128, random_state=42)"},"metadata":{}}]},{"cell_type":"markdown","source":"## Train dimension reduction","metadata":{}},{"cell_type":"code","source":"%%time\ntrain = scipy.sparse.load_npz(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_inputs_values.sparse.npz\")\ntrain = train.astype('float16',copy = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:57:38.337263Z","iopub.execute_input":"2022-10-11T02:57:38.337597Z","iopub.status.idle":"2022-10-11T02:58:43.947831Z","shell.execute_reply.started":"2022-10-11T02:57:38.337567Z","shell.execute_reply":"2022-10-11T02:58:43.94637Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"CPU times: user 38.4 s, sys: 3.32 s, total: 41.8 s\nWall time: 1min 5s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\npca_train = TruncatedSVD(n_components=128, random_state=42)\ntrain = pca_train.fit_transform(train)","metadata":{"execution":{"iopub.status.busy":"2022-10-11T02:58:43.949511Z","iopub.execute_input":"2022-10-11T02:58:43.949958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, Y.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tensorflow Keras librairies","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Concatenate,BatchNormalization","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metrics","metadata":{}},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and parameters","metadata":{}},{"cell_type":"code","source":"plateau = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    factor=0.5,\n    patience=2,\n    verbose=0,\n    mode=\"min\",\n    min_delta=1e-7\n)\nes = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=1e-7,\n    patience=10,\n    verbose=0,\n    mode=\"min\",\n    restore_best_weights=True,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Architecture from AmbrosM used for CITEseq\n\ndef create_model():\n\n    activation = 'selu'\n    \n    inputs = Input(shape= train.shape[1])\n\n    x0 = Dense(units = 256, \n              activation = activation,\n             )(inputs)\n    \n    x1 = Dense(units = 256, \n              activation = activation,\n                )(x0)\n    \n    x2 = Dense(units = 256, \n              activation = activation,\n             )(x1)\n\n    \n    x3 = Dense(units = 128, \n              activation = activation,\n             )(x2)\n    \n    x = Concatenate()([\n                        x0, \n                        x1, \n                        x2, \n                        x3\n                      ])\n    \n    x = Dense(units =Y.shape[1], \n              activation = 'linear',\n             )(x)\n    \n    model = Model(inputs, x)\n    \n    return model\n\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nN_SPLIT = 5\nkf = KFold(n_splits=N_SPLIT, shuffle=True, random_state=42)\n\nfor fold,(idx_tr, idx_va) in enumerate(kf.split(train)):\n    \n    X_tr = train[idx_tr]\n    y_tr = Y[idx_tr]\n    \n    X_va = train[idx_va]\n    y_va = Y[idx_va] \n    \n    model = create_model()\n\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-2),\n                  loss = 'mse',\n                  metrics=None)\n    model.fit(X_tr,\n              y_tr,\n              validation_data=(X_va,y_va),\n              epochs =1000,\n              verbose = 0,\n              batch_size=256,\n              callbacks = [es,plateau]\n             )\n    pred = model.predict(X_va)\n    \n    print(f'\\n --------- FOLD {fold} -----------')\n    print(f'Mean squared error = {np.round(mean_squared_error(y_va,pred),2)}')\n   \n    filename = f\"model_{fold}\"\n    model.save(filename)\n    print('model saved :',filename)\n        \n    del X_tr,X_va,y_tr,y_va\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test prediction","metadata":{}},{"cell_type":"code","source":"%%time\nmulti_test_x = scipy.sparse.load_npz(\"../input/multimodal-single-cell-as-sparse-matrix/test_multi_inputs_values.sparse.npz\")\nmulti_test_x = pca_train.transform(multi_test_x)\nmulti_test_x.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros((multi_test_x.shape[0], 23418), dtype='float16')\n\nfor fold in range(N_SPLIT):\n    print(f'fold {fold} prediction')\n    model = tf.keras.models.load_model(f\"model_{fold}\")\n    preds += (model.predict(multi_test_x)@pca_target.components_)/N_SPLIT\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission preparation","metadata":{}},{"cell_type":"code","source":"%%time\neval_ids = pd.read_parquet(\"../input/multimodal-single-cell-as-sparse-matrix/evaluation.parquet\")\neval_ids.cell_id = eval_ids.cell_id.astype(pd.CategoricalDtype())\neval_ids.gene_id = eval_ids.gene_id.astype(pd.CategoricalDtype())\n\nsubmission = pd.Series(name='target',\n                       index=pd.MultiIndex.from_frame(eval_ids), \n                       dtype=np.float32)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ny_columns = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_targets_idxcol.npz\",\n                   allow_pickle=True)[\"columns\"]\n\ntest_index = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/test_multi_inputs_idxcol.npz\",\n                    allow_pickle=True)[\"index\"]\n\ncell_dict = dict((k,v) for v,k in enumerate(test_index)) \nassert len(cell_dict)  == len(test_index)\n\ngene_dict = dict((k,v) for v,k in enumerate(y_columns))\nassert len(gene_dict) == len(y_columns)\n\neval_ids_cell_num = eval_ids.cell_id.apply(lambda x:cell_dict.get(x, -1))\neval_ids_gene_num = eval_ids.gene_id.apply(lambda x:gene_dict.get(x, -1))\nvalid_multi_rows = (eval_ids_gene_num !=-1) & (eval_ids_cell_num!=-1)\n\nsubmission.iloc[valid_multi_rows] = preds[eval_ids_cell_num[valid_multi_rows].to_numpy(),\neval_ids_gene_num[valid_multi_rows].to_numpy()]\n\ndel eval_ids_cell_num, eval_ids_gene_num, valid_multi_rows, eval_ids, test_index, y_columns\ngc.collect()\n\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.reset_index(drop=True, inplace=True)\nsubmission.index.name = 'row_id'\n\ncite_submission = pd.read_csv(\"../input/targets-multiome-sparse-scaled/submission_cite.csv\")\ncite_submission = cite_submission.set_index(\"row_id\")\ncite_submission = cite_submission[\"target\"]\nsubmission[submission.isnull()] = cite_submission[submission.isnull()]\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission_keras_multiome.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del preds, Y, train, multi_test_x\ngc.collect()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble submission from with \n\nhttps://www.kaggle.com/code/vslaykovsky/lb-0-811-normalized-ensembles-for-pearson-s-r\n\nwith files from :\n\nthe current notebook +\n\nhttps://www.kaggle.com/code/sskknt/msci-citeseq-keras-quickstart-dropout/data\n\nhttps://www.kaggle.com/code/xiafire/lb-t15-msci-multiome-catboostregressor/data\n","metadata":{}},{"cell_type":"code","source":"def std(x):\n    return (x - np.mean(x)) / np.std(x)\n\ndef gen_std_submission(path, cell_ids):\n    \"\"\"\n    Standardize submission per cell_id\n    \"\"\"\n    df = pd.read_csv(path)\n    df['cell_id'] = cell_ids    \n    vals = []\n    for idx, g in tqdm(df.groupby('cell_id', sort=False), desc=f'Standardizing {path}', miniters=1000):\n        vals.append(std(g.target).values)\n    vals = np.concatenate(vals)\n    return vals\n\ndef gen_ensemble(technology):\n    ensemble = None\n    for path in tqdm([path for path in SUBMISSIONS.keys() if technology in path], desc='Process submission'):\n        weight = SUBMISSIONS[path]\n        if ensemble is None:\n            ensemble = gen_std_submission(path, cell_ids) * weight\n        else:\n            ensemble += gen_std_submission(path, cell_ids) * weight\n    return ensemble","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSIONS = {\n    'submission_keras_multiome.csv':0.6,\n    '../input/msci-citeseq-keras-quickstart-dropout/submission.csv': 0.2,         \n    '../input/lb-t15-msci-multiome-catboostregressor/submission.csv':0.2,\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_ids = pd.read_parquet('../input/multimodal-single-cell-as-sparse-matrix/evaluation.parquet').cell_id\n\nPRED_SEGMENTS = [(0, 6812820), (6812820, 65744180)]\nensemble = []\nfor tech, (from_idx, to_idx) in tqdm(list(zip(['citeseq', 'multiome'], PRED_SEGMENTS)), desc='Technology'):    \n    ensemble.append(gen_ensemble(tech)[from_idx: to_idx])\n    \n    \nensemble = np.concatenate(ensemble)\n\ndf_submit = pd.read_parquet('../input/multimodal-single-cell-as-sparse-matrix/sample_submission.parquet')\ndf_submit['target'] = ensemble\ndf_submit.to_csv('submission_06_02_02_v1.csv', index=False)\ndf_submit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}