{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## KERAS for CITEseq & Multiome\n\nThis notebook is an update of MSCI CITEseq Keras Quickstart by AmbrosM :\nhttps://www.kaggle.com/code/ambrosm/msci-citeseq-keras-quickstart\n\nMy apologies for the Kagglers not mentioned.","metadata":{}},{"cell_type":"markdown","source":"<p> I'm a postgraduate student and it's my first time to join the Kaggle competition, so if my organization in this nootbook was not clear and you want to know more other information, you can comment below or send me a private message. As a result of I didn't control the time well, so the final plans were not been finished and submitted. The feature engineer's parts are not organized well so now I just to share my best NN model structure, though the final TruncaredSVD's parameters were changed a little.</p>","metadata":{}},{"cell_type":"code","source":"! pip install tables","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:51:31.903598Z","iopub.execute_input":"2023-03-14T07:51:31.904164Z","iopub.status.idle":"2023-03-14T07:51:42.981091Z","shell.execute_reply.started":"2023-03-14T07:51:31.904043Z","shell.execute_reply":"2023-03-14T07:51:42.979826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, pickle, datetime, scipy.sparse\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom colorama import Fore, Back, Style\n\nfrom sklearn.model_selection import GroupKFold,KFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import TruncatedSVD,PCA\nfrom sklearn.metrics import mean_squared_error\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\n\nimport scipy.sparse\n\nDATA_DIR = \"../input/open-problems-multimodal\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")\n\nVERBOSE = 0","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-14T07:51:42.983996Z","iopub.execute_input":"2023-03-14T07:51:42.985228Z","iopub.status.idle":"2023-03-14T07:51:44.117834Z","shell.execute_reply.started":"2023-03-14T07:51:42.985158Z","shell.execute_reply":"2023-03-14T07:51:44.116699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ------ CITEseq MODEL ---------","metadata":{}},{"cell_type":"markdown","source":"## Important features which are selected by myself","metadata":{}},{"cell_type":"code","source":"# important_cols =['ENSG00000004468',\n#  'ENSG00000005961',\n#  'ENSG00000010278',\n#  'ENSG00000010610',\n#  'ENSG00000026103',\n#  'ENSG00000026508',\n#  'ENSG00000072274',\n#  'ENSG00000073008',\n#  'ENSG00000077238',\n#  'ENSG00000081237',\n#  'ENSG00000083457',\n#  'ENSG00000085117',\n#  'ENSG00000091409',\n#  'ENSG00000100031',\n#  'ENSG00000101017',\n#  'ENSG00000105383',\n#  'ENSG00000106991',\n#  'ENSG00000110651',\n#  'ENSG00000110848',\n#  'ENSG00000110876',\n#  'ENSG00000112149',\n#  'ENSG00000115232',\n#  'ENSG00000117091',\n#  'ENSG00000125810',\n#  'ENSG00000130202',\n#  'ENSG00000135218',\n#  'ENSG00000135404',\n#  'ENSG00000137101',\n#  'ENSG00000139187',\n#  'ENSG00000143226',\n#  'ENSG00000150093',\n#  'ENSG00000150337',\n#  'ENSG00000150637',\n#  'ENSG00000157873',\n#  'ENSG00000160255',\n#  'ENSG00000164171',\n#  'ENSG00000166825',\n#  'ENSG00000169442',\n#  'ENSG00000173762',\n#  'ENSG00000179639',\n#  'ENSG00000185291',\n#  'ENSG00000185896',\n#  'ENSG00000186827',\n#  'ENSG00000188404',\n#  'ENSG00000196776',\n#  'ENSG00000197635',\n#  'ENSG00000198053',\n#  'ENSG00000203710',\n#  'ENSG00000204287',\n#  'ENSG00000204592',\n#  'ENSG00000206503',\n#  'ENSG00000211898',\n#  'ENSG00000211899',\n#  'ENSG00000261371']\n\n# print('important columns ',len(important_cols))","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:51:44.119479Z","iopub.execute_input":"2023-03-14T07:51:44.119851Z","iopub.status.idle":"2023-03-14T07:51:44.126613Z","shell.execute_reply.started":"2023-03-14T07:51:44.119814Z","shell.execute_reply":"2023-03-14T07:51:44.125498Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparation of the cross validation by batch","metadata":{}},{"cell_type":"code","source":"metadata_df = pd.read_csv(FP_CELL_METADATA, index_col = 'cell_id')\nmetadata_df = metadata_df[metadata_df.technology == \"citeseq\"]\n\n# Read train and convert to sparse matrix\nX = pd.read_hdf(FP_CITE_TRAIN_INPUTS)\nX.columns = [t.split('_')[0] for t in X.columns]\ncell_index = X.index\nmeta = metadata_df.reindex(cell_index)\n#X0 = X[important_cols].values\n\ndel X\ngc.collect()\n\n\n# Read test and convert to sparse matrix\nXt = pd.read_hdf(FP_CITE_TEST_INPUTS)\nXt.columns = [t.split('_')[0] for t in Xt.columns]\ncell_index_test = Xt.index\nmeta_test = metadata_df.reindex(cell_index_test)\n#X0t = Xt[important_cols].values\n\ndel Xt\ngc.collect()\n\n#print(f'X0 shape {X0.shape} X0t shape {X0t.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:51:44.129940Z","iopub.execute_input":"2023-03-14T07:51:44.130576Z","iopub.status.idle":"2023-03-14T07:53:12.449150Z","shell.execute_reply.started":"2023-03-14T07:51:44.130547Z","shell.execute_reply":"2023-03-14T07:53:12.447891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target normalization","metadata":{}},{"cell_type":"code","source":"Y = pd.read_hdf(FP_CITE_TRAIN_TARGETS)\nY = Y.values\nY -= Y.mean(axis=1).reshape(-1, 1)\nY /= Y.std(axis=1).reshape(-1, 1)\nY = Y.astype('float32', copy=True)\nY.shape\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:12.451733Z","iopub.execute_input":"2023-03-14T07:53:12.452110Z","iopub.status.idle":"2023-03-14T07:53:13.196227Z","shell.execute_reply.started":"2023-03-14T07:53:12.452073Z","shell.execute_reply":"2023-03-14T07:53:13.195115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Featrues was made by DCA(SVD), FA, scVi and Important columns which were selected by myself.","metadata":{}},{"cell_type":"code","source":"X = np.load(\"../input/citeseq-keras-1/citeseq_keras_train_data.npy\")\nXt = np.load(\"../input/citeseq-keras-1/citeseq_keras_test_data.npy\")\nprint(Xt.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:13.197848Z","iopub.execute_input":"2023-03-14T07:53:13.198450Z","iopub.status.idle":"2023-03-14T07:53:17.250258Z","shell.execute_reply.started":"2023-03-14T07:53:13.198413Z","shell.execute_reply":"2023-03-14T07:53:17.249248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tensorflow Keras librairies","metadata":{}},{"cell_type":"code","source":"import math\nfrom tensorflow import keras\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping\nfrom tensorflow.keras.layers import Dense, Input, Concatenate, Dropout, BatchNormalization,GaussianDropout\nfrom tensorflow.keras.utils import plot_model","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:17.251762Z","iopub.execute_input":"2023-03-14T07:53:17.252145Z","iopub.status.idle":"2023-03-14T07:53:21.999893Z","shell.execute_reply.started":"2023-03-14T07:53:17.252107Z","shell.execute_reply":"2023-03-14T07:53:21.998838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric and loss function","metadata":{}},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n\ndef negative_correlation_loss(y_true, y_pred):\n    my = K.mean(tf.convert_to_tensor(y_pred), axis=1)\n    my = tf.tile(tf.expand_dims(my, axis=1), (1, y_true.shape[1]))\n    ym = y_pred - my\n    r_num = K.sum(tf.multiply(y_true, ym), axis=1)\n    r_den = tf.sqrt(K.sum(K.square(ym), axis=1) * float(y_true.shape[-1]))\n    r = tf.reduce_mean(r_num / r_den)\n    return - r","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:22.001399Z","iopub.execute_input":"2023-03-14T07:53:22.002118Z","iopub.status.idle":"2023-03-14T07:53:22.012663Z","shell.execute_reply.started":"2023-03-14T07:53:22.002079Z","shell.execute_reply":"2023-03-14T07:53:22.011574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and parameters","metadata":{}},{"cell_type":"code","source":"LR_START = 0.01\nBATCH_SIZE = 512\n\ndef create_model():\n    #      680, 256, 300, 140, \n    reg1 = 9.613e-06\n    reg2 = 1e-07\n    REG1 = tf.keras.regularizers.l2(reg1)\n    REG2 = tf.keras.regularizers.l2(reg2)\n    DROP = 0.15\n    DROP2 = 0.05\n    #512, 380, 620, 350, 0.15, 0.05\n    activation = 'selu'\n    inputs = Input(shape =(100,))\n\n    x0 = Dense(512, \n              kernel_regularizer = REG1,\n              activation = activation,\n             )(inputs)\n    x0 = GaussianDropout(DROP)(x0)\n    \n    \n    x1 = Dense(380, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x0)\n    x1 = GaussianDropout(DROP2)(x1)\n    \n    \n    x2 = Dense(620, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x1) \n    x2= GaussianDropout(DROP2)(x2)\n    \n    x3 = Dense(350,\n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x2)\n    x3 = GaussianDropout(DROP2)(x3)\n\n         \n    x = Concatenate()([\n                x0, \n                x1, \n                x2, \n                x3\n                ])\n    \n    x = Dense(Y.shape[1], \n                kernel_regularizer = REG2,\n                activation='linear',\n                )(x)\n    \n    \n    model = Model(inputs, x)\n    model.summary()\n\n    return model\ndisplay(plot_model(create_model(), show_layer_names=True, show_shapes=True, dpi=64))","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:22.015076Z","iopub.execute_input":"2023-03-14T07:53:22.016117Z","iopub.status.idle":"2023-03-14T07:53:25.339243Z","shell.execute_reply.started":"2023-03-14T07:53:22.016088Z","shell.execute_reply":"2023-03-14T07:53:25.337900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_model()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:25.344361Z","iopub.execute_input":"2023-03-14T07:53:25.344717Z","iopub.status.idle":"2023-03-14T07:53:25.404846Z","shell.execute_reply.started":"2023-03-14T07:53:25.344686Z","shell.execute_reply":"2023-03-14T07:53:25.403670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 0.01\nBATCH_SIZE = 512\n\ndef create_model_multi():\n    reg1 = 9.613e-06\n    reg2 = 1e-07\n    REG1 = tf.keras.regularizers.l2(reg1)\n    REG2 = tf.keras.regularizers.l2(reg2)\n    DROP = 0.1\n    DROP2 = 0.1\n    #512, 380, 620, 350, 0.15, 0.05\n    activation = 'selu'\n    inputs = Input(shape =(X.shape[1],))\n\n    x0 = Dense(256, \n              kernel_regularizer = REG1,\n              activation = activation,\n             )(inputs)\n    x0 = Dropout(DROP)(x0)\n    \n    \n    x1 = Dense(512, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x0)\n    x1 = Dropout(DROP2)(x1)\n    \n    \n    x2 = Dense(512, \n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x1) \n    x2= Dropout(DROP2)(x2)\n    \n    x3 = Dense(140,\n               kernel_regularizer = REG1,\n               activation = activation,\n             )(x2)\n    x3 = Dropout(DROP2)(x3)\n\n         \n    x = Concatenate()([\n                x0, \n                x1, \n                x2, \n                x3\n                ])\n    \n    x = Dense(Y.shape[1], \n                kernel_regularizer = REG2,\n                activation='linear',\n                )(x)\n    \n    \n    model = Model(inputs, x)\n    \n\n    return model\ndisplay(plot_model(create_model_multi(), show_layer_names=True, show_shapes=True, dpi=64))","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:25.406540Z","iopub.execute_input":"2023-03-14T07:53:25.407167Z","iopub.status.idle":"2023-03-14T07:53:25.628413Z","shell.execute_reply.started":"2023-03-14T07:53:25.407128Z","shell.execute_reply":"2023-03-14T07:53:25.627154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"meta[\"batch\"] = meta[\"day\"].astype(str) + \"_\" + meta[\"donor\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:25.630600Z","iopub.execute_input":"2023-03-14T07:53:25.631283Z","iopub.status.idle":"2023-03-14T07:53:25.723675Z","shell.execute_reply.started":"2023-03-14T07:53:25.631240Z","shell.execute_reply":"2023-03-14T07:53:25.722627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.shape, X.shape,Y.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:25.725130Z","iopub.execute_input":"2023-03-14T07:53:25.725528Z","iopub.status.idle":"2023-03-14T07:53:25.735027Z","shell.execute_reply.started":"2023-03-14T07:53:25.725486Z","shell.execute_reply":"2023-03-14T07:53:25.733827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nEPOCHS = 600 \nN_SPLITS = 9\n\npred_train = np.zeros((Y.shape[0],Y.shape[1]))\n\n\n#kf = GroupKFold(n_splits=N_SPLITS)\nkf = KFold(n_splits=N_SPLITS,shuffle=True)\nscore_list = []\n\n#folds = []\n# for i in [415,150,300,512,350]:\nnp.random.seed(1)\ntf.random.set_seed(1)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(X)):#, groups=meta.batch\n    start_time = datetime.datetime.now()\n    model = None\n    gc.collect()\n\n    X_tr = X[idx_tr]\n    y_tr = Y[idx_tr]\n    #c_tr =ct[idx_tr]\n    #s_tr =st[idx_tr]\n    X_va = X[idx_va]\n    y_va = Y[idx_va]\n    #c_va =ct[idx_va]\n    #s_va =st[idx_va]\n    print(meta[\"batch\"][idx_va].iloc[0])\n    lr = ReduceLROnPlateau(\n                    monitor = \"val_loss\",\n                    factor = 0.9, \n                    patience = 4, \n                    verbose = VERBOSE)\n\n    es = EarlyStopping(\n                    monitor = \"val_loss\",\n                    patience = 40, \n                    verbose = VERBOSE,\n                    mode = \"min\", \n                    restore_best_weights = True)\n\n    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n                    filepath = './citeseq',\n                    save_weights_only = True,\n                    monitor = 'val_loss',\n                    mode = 'min',\n                    save_best_only = True)\n\n    callbacks = [\n                    lr, \n                    es, \n                    model_checkpoint_callback\n                    ]\n\n    model = create_model_multi()#create_model()#(True, False, False, False, False, True, True, False, False, False, False, True, True)\n\n    model.compile(\n                optimizer = tf.keras.optimizers.Adam(learning_rate=LR_START),\n                metrics = [negative_correlation_loss],#None,#\n                loss = negative_correlation_loss#'mse'#\n                 )\n    # Training\n    model.fit(\n                X_tr, #s_tr,\n                y_tr, \n                validation_data=(\n                                X_va, #s_va,\n                                y_va), \n                epochs = EPOCHS,\n                verbose = VERBOSE,\n                batch_size = BATCH_SIZE,\n                shuffle = True,\n                callbacks = callbacks)\n\n    del X_tr, y_tr \n    gc.collect()\n\n    model.load_weights('./citeseq')\n    #if meta[\"donor\"][idx_va][0] != 13176:\n    model.save(f\"./submissions/model_{fold}\")\n        #folds.append(fold)\n    print('model saved')\n    #else:\n    #    print('model don\\'t need to be saved')\n\n    #  Model validation\n    y_va_pred = model.predict(X_va) #s_va,\n    corrscore = correlation_score(y_va, y_va_pred)\n    pred_train[idx_va] = y_va_pred\n    #print(meta[\"batch\"][idx_va][0])\n    print(f\"Fold {fold}, correlation =  {corrscore:.5f}\")\n    del X_va, y_va, y_va_pred\n    gc.collect()\n    score_list.append(corrscore)\n\n# Show overall score\nprint(f\"{Fore.GREEN}{Style.BRIGHT}Mean corr = {np.array(score_list).mean():.5f}{Style.RESET_ALL}\")\nscore_total = correlation_score(Y, pred_train)\nprint(f\"{Fore.BLUE}{Style.BRIGHT}Oof corr   = {score_total:.5f}{Style.RESET_ALL}\")\n#Mean corr = 0.90327\n#Oof corr   = 0.90242","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:53:25.737192Z","iopub.execute_input":"2023-03-14T07:53:25.737930Z","iopub.status.idle":"2023-03-14T08:01:12.867064Z","shell.execute_reply.started":"2023-03-14T07:53:25.737894Z","shell.execute_reply":"2023-03-14T08:01:12.865750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CITEseq Test prediction","metadata":{}},{"cell_type":"code","source":"Xt.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:12.868961Z","iopub.execute_input":"2023-03-14T08:01:12.869756Z","iopub.status.idle":"2023-03-14T08:01:12.877777Z","shell.execute_reply.started":"2023-03-14T08:01:12.869710Z","shell.execute_reply":"2023-03-14T08:01:12.876526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.zeros((len(Xt), 140), dtype=np.float32)\nfor fold in range(N_SPLITS):\n    print(f\"Predicting with fold {fold}\")\n    model = load_model(f\"./submissions/model_{fold}\",\n                       custom_objects={'negative_correlation_loss': negative_correlation_loss})\n    test_pred += model.predict(Xt)#@svd_y.components_\n\n# Copy the targets for the data leak but useless since the change in the public LB...\n#test_pred[:7476] = Y[:7476]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:12.879310Z","iopub.execute_input":"2023-03-14T08:01:12.880423Z","iopub.status.idle":"2023-03-14T08:01:19.468677Z","shell.execute_reply.started":"2023-03-14T08:01:12.880380Z","shell.execute_reply":"2023-03-14T08:01:19.465246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xt.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:41:59.759735Z","iopub.execute_input":"2023-03-14T08:41:59.760241Z","iopub.status.idle":"2023-03-14T08:41:59.767473Z","shell.execute_reply.started":"2023-03-14T08:41:59.760196Z","shell.execute_reply":"2023-03-14T08:41:59.766408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:41:46.608782Z","iopub.execute_input":"2023-03-14T08:41:46.609213Z","iopub.status.idle":"2023-03-14T08:41:46.619461Z","shell.execute_reply.started":"2023-03-14T08:41:46.609159Z","shell.execute_reply":"2023-03-14T08:41:46.618189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ------ Multiome MODEL ---------","metadata":{}},{"cell_type":"markdown","source":"## Upload of the multiome files after TruncatedSVD","metadata":{}},{"cell_type":"code","source":"# with open('../input/targets-multiome-sparse-scaled/INDEX_train_multiome.pkl','rb') as f: INDEX_train_multiome = pickle.load(f)\n# with open('../input/targets-multiome-sparse-scaled/train_512.pkl','rb') as f: X = pickle.load(f)\n# #with open('../input/targets-multiome-sparse-scaled/pca_train_512.pkl','rb') as f: pca_train = pickle.load(f)\n# with open('../input/targets-multiome-sparse-scaled/pca_target_512.pkl','rb') as f: pca_target = pickle.load(f)\n# with open('../input/targets-multiome-sparse-scaled/Y_512.pkl','rb') as f: Y = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.470486Z","iopub.execute_input":"2023-03-14T08:01:19.470887Z","iopub.status.idle":"2023-03-14T08:01:19.477854Z","shell.execute_reply.started":"2023-03-14T08:01:19.470847Z","shell.execute_reply":"2023-03-14T08:01:19.475472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = X[:,:40]\n# X.shape\n# # X -= X.mean(axis=0).reshape(1, -1)\n# # X /= X.std(axis=0).reshape(1, -1)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.480314Z","iopub.execute_input":"2023-03-14T08:01:19.480700Z","iopub.status.idle":"2023-03-14T08:01:19.493067Z","shell.execute_reply.started":"2023-03-14T08:01:19.480663Z","shell.execute_reply":"2023-03-14T08:01:19.491751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# metadata_df = pd.read_csv('../input/open-problems-multimodal/metadata.csv',index_col='cell_id')\n# metadata_df = metadata_df[metadata_df.technology==\"multiome\"]\n# meta = metadata_df.reindex(INDEX_train_multiome)\n# del metadata_df\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.494567Z","iopub.execute_input":"2023-03-14T08:01:19.494991Z","iopub.status.idle":"2023-03-14T08:01:19.505069Z","shell.execute_reply.started":"2023-03-14T08:01:19.494957Z","shell.execute_reply":"2023-03-14T08:01:19.503820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# meta[\"batch\"] = meta[\"day\"].astype(str) + \"_\" + meta[\"donor\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.506653Z","iopub.execute_input":"2023-03-14T08:01:19.507510Z","iopub.status.idle":"2023-03-14T08:01:19.517386Z","shell.execute_reply.started":"2023-03-14T08:01:19.507473Z","shell.execute_reply":"2023-03-14T08:01:19.516072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import scipy.sparse\n# train_targets = scipy.sparse.load_npz(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_targets_values.sparse.npz\")\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.518693Z","iopub.execute_input":"2023-03-14T08:01:19.519097Z","iopub.status.idle":"2023-03-14T08:01:19.530163Z","shell.execute_reply.started":"2023-03-14T08:01:19.519063Z","shell.execute_reply":"2023-03-14T08:01:19.528853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training for Multiome","metadata":{}},{"cell_type":"code","source":"# %%time\n# import warnings\n# warnings.filterwarnings(\"ignore\")\n\n# N_SPLIT = 3\n# kf = KFold(n_splits=N_SPLIT, shuffle=True, random_state=42)\n# #kf = GroupKFold(n_splits = N_SPLIT)\n# pred_train = np.zeros((Y.shape[0],Y.shape[1]))\n# for fold,(idx_tr, idx_va) in enumerate(kf.split(X)): #,groups=meta.batch\n    \n#     X_tr = X[idx_tr]\n#     y_tr = Y[idx_tr]\n    \n#     X_va = X[idx_va]\n#     y_va = Y[idx_va]\n#     train_targets_va = train_targets[idx_va].toarray()\n    \n#     model = create_model_multi()\n    \n#     lr = ReduceLROnPlateau(\n#                 monitor = \"val_loss\",\n#                 factor = 0.9, \n#                 patience = 4, \n#                 verbose = VERBOSE)\n    \n#     es = EarlyStopping(\n#                 monitor = \"val_loss\",\n#                 patience = 30, \n#                 verbose = VERBOSE,\n#                 mode = \"min\", \n#                 restore_best_weights = True)\n\n#     model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n#                   metrics = None,\n#                   loss = 'mse')\n#     model.fit(X_tr,\n#               y_tr,\n#               validation_data=(X_va,y_va),\n#               epochs =500,\n#               verbose = VERBOSE,\n#               batch_size=256,\n#               callbacks = [es,lr]\n#              )\n#     pred = model.predict(X_va)#@pca_target.components_\n#     pred_train[idx_va] += pred\n#     #train_targets_va\n#     #corrscore = correlation_score(y_va, y_va_pred)\n    \n#     print(f'\\n --------- FOLD {fold} -----------')\n#     print(f'corrscore = {np.round(correlation_score(y_va,pred),5)}')\n#     print(f'corrscore = {np.round(correlation_score(train_targets_va,pred@pca_target.components_),5)}')\n   \n#     filename = f\"m_model_{fold}\"\n#     model.save(filename)\n#     print('model saved :',filename)\n        \n#     del X_tr,X_va,y_tr,y_va\n#     gc.collect()\n\n# score_total = correlation_score(train_targets.toarray(), pred_train@pca_target.components_)\n# print(f\"{Fore.BLUE}{Style.BRIGHT}Oof corr   = {score_total:.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.531553Z","iopub.execute_input":"2023-03-14T08:01:19.531984Z","iopub.status.idle":"2023-03-14T08:01:19.542838Z","shell.execute_reply.started":"2023-03-14T08:01:19.531949Z","shell.execute_reply":"2023-03-14T08:01:19.541643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test predictions for Multiome","metadata":{}},{"cell_type":"code","source":"# multi_test_x = scipy.sparse.load_npz(\"../input/multimodal-single-cell-as-sparse-matrix/test_multi_inputs_values.sparse.npz\")\n# multi_test_x = pca_train.transform(multi_test_x)\n# #multi_test_x = both[105942:]\n# multi_test_x = multi_test_x[:,:40]\n# multi_test_x.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.544526Z","iopub.execute_input":"2023-03-14T08:01:19.544882Z","iopub.status.idle":"2023-03-14T08:01:19.558636Z","shell.execute_reply.started":"2023-03-14T08:01:19.544849Z","shell.execute_reply":"2023-03-14T08:01:19.557449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del X,Y,INDEX_train_multiome\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.560196Z","iopub.execute_input":"2023-03-14T08:01:19.560610Z","iopub.status.idle":"2023-03-14T08:01:19.571424Z","shell.execute_reply.started":"2023-03-14T08:01:19.560575Z","shell.execute_reply":"2023-03-14T08:01:19.570006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = np.zeros((multi_test_x.shape[0], 23418), dtype='float16')\n\n# for fold in range(N_SPLIT):\n#     print(f'fold {fold} prediction')\n#     model = load_model(f\"m_model_{fold}\",\n#                        custom_objects={'negative_correlation_loss': negative_correlation_loss})\n#     preds += (model.predict(multi_test_x)@pca_target.components_)/N_SPLIT\n\n#     gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.572910Z","iopub.execute_input":"2023-03-14T08:01:19.573347Z","iopub.status.idle":"2023-03-14T08:01:19.583168Z","shell.execute_reply.started":"2023-03-14T08:01:19.573306Z","shell.execute_reply":"2023-03-14T08:01:19.582103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eval_ids = pd.read_parquet(\"../input/multimodal-single-cell-as-sparse-matrix/evaluation.parquet\")\n# eval_ids.cell_id = eval_ids.cell_id.astype(pd.CategoricalDtype())\n# eval_ids.gene_id = eval_ids.gene_id.astype(pd.CategoricalDtype())\n\n# submission = pd.Series(name='target',\n#                        index=pd.MultiIndex.from_frame(eval_ids), \n#                        dtype=np.float32)\n# submission","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.584595Z","iopub.execute_input":"2023-03-14T08:01:19.585019Z","iopub.status.idle":"2023-03-14T08:01:19.598219Z","shell.execute_reply.started":"2023-03-14T08:01:19.584971Z","shell.execute_reply":"2023-03-14T08:01:19.597108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_columns = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_targets_idxcol.npz\",\n#                    allow_pickle=True)[\"columns\"]\n\n# test_index = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/test_multi_inputs_idxcol.npz\",\n#                     allow_pickle=True)[\"index\"]\n\n# cell_dict = dict((k,v) for v,k in enumerate(test_index)) \n# assert len(cell_dict)  == len(test_index)\n\n# gene_dict = dict((k,v) for v,k in enumerate(y_columns))\n# assert len(gene_dict) == len(y_columns)\n\n# eval_ids_cell_num = eval_ids.cell_id.apply(lambda x:cell_dict.get(x, -1))\n# eval_ids_gene_num = eval_ids.gene_id.apply(lambda x:gene_dict.get(x, -1))\n# valid_multi_rows = (eval_ids_gene_num !=-1) & (eval_ids_cell_num!=-1)\n\n# submission.iloc[valid_multi_rows] = preds[eval_ids_cell_num[valid_multi_rows].to_numpy(),\n# eval_ids_gene_num[valid_multi_rows].to_numpy()]\n\n# del eval_ids_cell_num, eval_ids_gene_num, valid_multi_rows, eval_ids, test_index, y_columns\n# gc.collect()\n\n# submission","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.600213Z","iopub.execute_input":"2023-03-14T08:01:19.600538Z","iopub.status.idle":"2023-03-14T08:01:19.609597Z","shell.execute_reply.started":"2023-03-14T08:01:19.600504Z","shell.execute_reply":"2023-03-14T08:01:19.607988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Total submission","metadata":{}},{"cell_type":"code","source":"submission1 = pd.read_csv(\"/kaggle/input/2nd-place-gru-cite/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:19.619927Z","iopub.execute_input":"2023-03-14T08:01:19.620278Z","iopub.status.idle":"2023-03-14T08:01:58.008119Z","shell.execute_reply.started":"2023-03-14T08:01:19.620245Z","shell.execute_reply":"2023-03-14T08:01:58.006997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred.ravel().shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:39:37.505134Z","iopub.execute_input":"2023-03-14T08:39:37.505675Z","iopub.status.idle":"2023-03-14T08:39:37.516298Z","shell.execute_reply.started":"2023-03-14T08:39:37.505632Z","shell.execute_reply":"2023-03-14T08:39:37.515165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"7476*140 + 5766180","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:40:37.810542Z","iopub.execute_input":"2023-03-14T08:40:37.811015Z","iopub.status.idle":"2023-03-14T08:40:37.821816Z","shell.execute_reply.started":"2023-03-14T08:40:37.810975Z","shell.execute_reply":"2023-03-14T08:40:37.820764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission1[\"target\"].iloc[7476*140:6812820] = test_pred.ravel()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:40:50.646480Z","iopub.execute_input":"2023-03-14T08:40:50.647445Z","iopub.status.idle":"2023-03-14T08:40:50.663807Z","shell.execute_reply.started":"2023-03-14T08:40:50.647404Z","shell.execute_reply":"2023-03-14T08:40:50.662694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission1[\"target\"]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:40:56.434597Z","iopub.execute_input":"2023-03-14T08:40:56.434988Z","iopub.status.idle":"2023-03-14T08:40:56.483029Z","shell.execute_reply.started":"2023-03-14T08:40:56.434957Z","shell.execute_reply":"2023-03-14T08:40:56.481606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.reset_index(drop=True, inplace=True)\n# submission.index.name = 'row_id'\n\n# cite_submission = pd.read_csv(\"submission_lolo_1.csv\")\n# cite_submission = cite_submission.set_index(\"row_id\")\n# cite_submission = cite_submission[\"target\"]\n# submission[submission.isnull()] = cite_submission[submission.isnull()]\n# submission","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.316159Z","iopub.status.idle":"2023-03-14T08:01:58.317252Z","shell.execute_reply.started":"2023-03-14T08:01:58.316955Z","shell.execute_reply":"2023-03-14T08:01:58.316982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del cite_submission\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.318466Z","iopub.status.idle":"2023-03-14T08:01:58.319498Z","shell.execute_reply.started":"2023-03-14T08:01:58.319209Z","shell.execute_reply":"2023-03-14T08:01:58.319239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_ensembling = pd.read_csv('../input/open-problems-multimodal/sample_submission.csv')\n# submission1 = sub_ensembling.copy()\n# submission1['target'] = submission\n# submission1","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.320933Z","iopub.status.idle":"2023-03-14T08:01:58.321826Z","shell.execute_reply.started":"2023-03-14T08:01:58.321554Z","shell.execute_reply":"2023-03-14T08:01:58.321579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission1.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.323269Z","iopub.status.idle":"2023-03-14T08:01:58.324130Z","shell.execute_reply.started":"2023-03-14T08:01:58.323863Z","shell.execute_reply":"2023-03-14T08:01:58.323888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# sub_preds = test_pred.reshape(-1)\n# multi_preds = np.load('/kaggle/input/single-cell-features/senkin_multi_predictions.npy')\n\n# sub = pd.read_csv('../input/open-problems-multimodal/sample_submission.csv')\n# sub['target'] = np.concatenate([sub_preds,multi_preds])\n# sub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.325566Z","iopub.status.idle":"2023-03-14T08:01:58.326456Z","shell.execute_reply.started":"2023-03-14T08:01:58.326166Z","shell.execute_reply":"2023-03-14T08:01:58.326213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_preds.shape[0] + multi_preds.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.331261Z","iopub.status.idle":"2023-03-14T08:01:58.331760Z","shell.execute_reply.started":"2023-03-14T08:01:58.331494Z","shell.execute_reply":"2023-03-14T08:01:58.331516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# multi_preds.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-14T08:01:58.333731Z","iopub.status.idle":"2023-03-14T08:01:58.334237Z","shell.execute_reply.started":"2023-03-14T08:01:58.333965Z","shell.execute_reply":"2023-03-14T08:01:58.333988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}