{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary","metadata":{"papermill":{"duration":0.00899,"end_time":"2022-09-01T07:08:29.620468","exception":false,"start_time":"2022-09-01T07:08:29.611478","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"[Simple Submission - Average by gene_id](https://www.kaggle.com/code/shuntarotanaka/simple-submission-average-by-gene-id) scored 0.741 \nThis note book calculates average of gene_id grouped by the day.\nBut the score is 0.748.\nDay may not affect the results much.<br>\nLooking at CV, citeseq improves 0.0035 and multiome improves 0.0068 in correlation if days are taken into account.\n","metadata":{}},{"cell_type":"code","source":"pip install tables","metadata":{"papermill":{"duration":13.416536,"end_time":"2022-09-01T07:08:43.046238","exception":false,"start_time":"2022-09-01T07:08:29.629702","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-06T01:03:45.854412Z","iopub.execute_input":"2022-09-06T01:03:45.854901Z","iopub.status.idle":"2022-09-06T01:03:59.833906Z","shell.execute_reply.started":"2022-09-06T01:03:45.854757Z","shell.execute_reply":"2022-09-06T01:03:59.832434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, pickle\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom colorama import Fore, Back, Style\nfrom matplotlib.ticker import MaxNLocator\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, scale\nfrom sklearn.decomposition import PCA\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.linear_model import Ridge, LinearRegression\nfrom sklearn.metrics import mean_squared_error\n\nfrom scipy.sparse import *\nfrom scipy.sparse import linalg\nfrom sklearn.decomposition import TruncatedSVD\nfrom scipy import interpolate\nimport random as rd\n\nimport psutil\n\ntry:\n    os.environ['SATURN_IMAGE']\n    kernel = \"saturn_cloud\"\nexcept KeyError:\n    kernel = \"kaggle\"\n\nDATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nif kernel==\"saturn_cloud\":\n    SPARSE_DIR = \"./\"\n    META_DIR = \"./\"\nelse: # kaggle\n#    SPARSE_DIR = \"../input/opmsci-sparse-data/\"\n    META_DIR = \"../input/open-problems-multimodal-singlecell-metadata/\"\n    SPARSE_DIR = \"../input/open-problems-multimodal-singlecell-metadata/\"\n\nMETA_CITE_TRAIN = os.path.join(META_DIR,\"cite_train_meta.csv\")\nMETA_CITE_TEST = os.path.join(META_DIR,\"cite_test_meta.csv\")\nMETA_MULTI_TRAIN = os.path.join(META_DIR,\"multiome_train_meta.csv\")\nMETA_MULTI_TEST = os.path.join(META_DIR,\"multiome_test_meta.csv\")\n\nSPS_CITE_TRAIN_INPUTS = os.path.join(SPARSE_DIR,\"train_cite_inputs.npz\")\nSPS_CITE_TRAIN_TARGETS = os.path.join(SPARSE_DIR,\"train_cite_targets.npz\")\nSPS_CITE_TEST_INPUTS = os.path.join(SPARSE_DIR,\"test_cite_inputs.npz\")\n\nSPS_MULTIOME_TRAIN_INPUTS = os.path.join(SPARSE_DIR,\"train_multi_inputs.npz\")\nSPS_MULTIOME_TRAIN_TARGETS = os.path.join(SPARSE_DIR,\"train_multi_targets.npz\")\nSPS_MULTIOME_TEST_INPUTS = os.path.join(SPARSE_DIR,\"test_multi_inputs.npz\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.111352,"end_time":"2022-09-01T07:08:44.169119","exception":false,"start_time":"2022-09-01T07:08:43.057767","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-09-06T01:03:59.836312Z","iopub.execute_input":"2022-09-06T01:03:59.836701Z","iopub.status.idle":"2022-09-06T01:04:00.517418Z","shell.execute_reply.started":"2022-09-06T01:03:59.836662Z","shell.execute_reply":"2022-09-06T01:04:00.516249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cite ","metadata":{"papermill":{"duration":0.010004,"end_time":"2022-09-01T07:08:44.189466","exception":false,"start_time":"2022-09-01T07:08:44.179462","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Read pre-sorted meta data ","metadata":{"papermill":{"duration":0.009923,"end_time":"2022-09-01T07:08:44.209622","exception":false,"start_time":"2022-09-01T07:08:44.199699","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ncite_ym = pd.read_csv(META_CITE_TRAIN)","metadata":{"papermill":{"duration":0.087675,"end_time":"2022-09-01T07:08:44.307485","exception":false,"start_time":"2022-09-01T07:08:44.21981","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:04:00.518686Z","iopub.execute_input":"2022-09-06T01:04:00.519027Z","iopub.status.idle":"2022-09-06T01:04:00.628841Z","shell.execute_reply.started":"2022-09-06T01:04:00.518992Z","shell.execute_reply":"2022-09-06T01:04:00.627595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read pre-converted sparse data<br />\nIt is faster and memory effective.","metadata":{"papermill":{"duration":0.01036,"end_time":"2022-09-01T07:08:44.328345","exception":false,"start_time":"2022-09-01T07:08:44.317985","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ncite_y = load_npz(SPS_CITE_TRAIN_TARGETS)","metadata":{"papermill":{"duration":1.392838,"end_time":"2022-09-01T07:08:45.731395","exception":false,"start_time":"2022-09-01T07:08:44.338557","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:04:00.632357Z","iopub.execute_input":"2022-09-06T01:04:00.633371Z","iopub.status.idle":"2022-09-06T01:04:02.177701Z","shell.execute_reply.started":"2022-09-06T01:04:00.633299Z","shell.execute_reply":"2022-09-06T01:04:02.176540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cross-Validation","metadata":{}},{"cell_type":"markdown","source":"Define Correlation score. This is from [ambrosm's msci-multiome-quickstart](http://https://www.kaggle.com/code/ambrosm/msci-multiome-quickstart)","metadata":{}},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    if y_true.shape != y_pred.shape: raise ValueError(\"Shapes are different.\")\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:02.181084Z","iopub.execute_input":"2022-09-06T01:04:02.181446Z","iopub.status.idle":"2022-09-06T01:04:02.188476Z","shell.execute_reply.started":"2022-09-06T01:04:02.181414Z","shell.execute_reply":"2022-09-06T01:04:02.187287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate CV for five folds","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=5,shuffle=True,random_state=1)\nscores = np.zeros((10,2))\nfor consider_day in [True,False]:\n    print(\"Day factor considered.\" if consider_day else \"Day factor ignored.\")\n    for fold,(idx_tr,idx_va) in enumerate(kf.split(cite_y)):\n        kf_tr_y,kf_tr_ym = cite_y[idx_tr],cite_ym.iloc[idx_tr]\n        kf_cite_all_means = kf_tr_y.mean(axis=0)\n        kf_cite_means = {}\n        kf_tr_ym.head()\n        for d in kf_tr_ym.day.unique():  ## Calculate the average of each day from training target\n            kf_cite_means[d] = kf_tr_y[kf_tr_ym.day==d,:].mean(axis=0)\n        kf_va_y,kf_va_ym = cite_y[idx_va],cite_ym.iloc[idx_va]\n        kf_va_ypred = np.zeros(kf_va_y.shape)\n        for day in kf_va_ym.day.unique():  ## Fill the validation y(target) prediction from this fold's training target\n            kf_va_ypred[kf_va_ym.day==day] = (kf_cite_means[day] if consider_day else kf_cite_all_means)\n        mse = mean_squared_error(kf_va_y.toarray(),kf_va_ypred)\n        corrscore = correlation_score(kf_va_y.toarray(),kf_va_ypred)\n        print(f\"Fold {fold}:mse = {mse:.5f},corr = {corrscore:.3f}\")\n        scores[fold + 5*(1 if consider_day else 0) ,:] = [mse,corrscore]","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:02.189946Z","iopub.execute_input":"2022-09-06T01:04:02.190390Z","iopub.status.idle":"2022-09-06T01:04:17.013023Z","shell.execute_reply.started":"2022-09-06T01:04:02.190348Z","shell.execute_reply":"2022-09-06T01:04:17.011856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_av = scores[0:5,:].mean(axis=0)\nday_av = scores[5:10,:].mean(axis=0)\nprint(\"All Average : mse=%f correlation=%f\"%(all_av[0],all_av[1]))\nprint(\"Day Average : mse=%f correlation=%f\"%(day_av[0],day_av[1]))\nprint(\"Improvement : mse=%f correlation=%f\"%(day_av[0]-all_av[0],day_av[1]-all_av[1]))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:17.014624Z","iopub.execute_input":"2022-09-06T01:04:17.015045Z","iopub.status.idle":"2022-09-06T01:04:17.022892Z","shell.execute_reply.started":"2022-09-06T01:04:17.015012Z","shell.execute_reply":"2022-09-06T01:04:17.021341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate means for submission from whole data","metadata":{}},{"cell_type":"markdown","source":"Calculate average of each gene_id on the same day from whole data.","metadata":{"papermill":{"duration":0.010945,"end_time":"2022-09-01T07:08:45.752882","exception":false,"start_time":"2022-09-01T07:08:45.741937","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ncite_means = {}\ncite_ym.head()\nfor d in cite_ym.day.unique():\n    cite_means[d] = cite_y[cite_ym.day==d,:].mean(axis=0)\n    ","metadata":{"papermill":{"duration":0.439393,"end_time":"2022-09-01T07:08:46.202738","exception":false,"start_time":"2022-09-01T07:08:45.763345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:04:17.024416Z","iopub.execute_input":"2022-09-06T01:04:17.024997Z","iopub.status.idle":"2022-09-06T01:04:17.132220Z","shell.execute_reply.started":"2022-09-06T01:04:17.024963Z","shell.execute_reply":"2022-09-06T01:04:17.131147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare for day 7(private test) with linear extrapolation of the other days. This is not used for public test.\nIf linear extrapolation works fine or not is a remaining issue.","metadata":{}},{"cell_type":"code","source":"cite_ym.day.unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:17.133532Z","iopub.execute_input":"2022-09-06T01:04:17.133856Z","iopub.status.idle":"2022-09-06T01:04:17.143413Z","shell.execute_reply.started":"2022-09-06T01:04:17.133827Z","shell.execute_reply":"2022-09-06T01:04:17.142533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_interp = interpolate.interp1d(np.array([2,3,4]).T,\n                                   np.concatenate((cite_means[2],cite_means[3],cite_means[4])),axis=0,fill_value='extrapolate')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:17.145030Z","iopub.execute_input":"2022-09-06T01:04:17.145486Z","iopub.status.idle":"2022-09-06T01:04:17.153937Z","shell.execute_reply.started":"2022-09-06T01:04:17.145454Z","shell.execute_reply":"2022-09-06T01:04:17.153138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_means[7] = cite_interp(7)\ncite_means[10] = cite_interp(10)\n##cite_means[7] = 0*cite_interp(7)   ## replace 0 to check LB score do not change\n##cite_means[10] = 0*cite_interp(10)   ## replace 0 to check LB score do not change","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:17.155194Z","iopub.execute_input":"2022-09-06T01:04:17.155527Z","iopub.status.idle":"2022-09-06T01:04:17.165174Z","shell.execute_reply.started":"2022-09-06T01:04:17.155496Z","shell.execute_reply":"2022-09-06T01:04:17.164056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Delete variables no longer needed to free memory","metadata":{}},{"cell_type":"code","source":"del cite_y,cite_ym,kf_tr_y,kf_tr_ym,kf_va_y,kf_va_ym,kf_va_ypred\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:17.166562Z","iopub.execute_input":"2022-09-06T01:04:17.166883Z","iopub.status.idle":"2022-09-06T01:04:17.295192Z","shell.execute_reply.started":"2022-09-06T01:04:17.166853Z","shell.execute_reply":"2022-09-06T01:04:17.294262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Multiome","metadata":{"papermill":{"duration":0.010559,"end_time":"2022-09-01T07:12:43.411379","exception":false,"start_time":"2022-09-01T07:12:43.40082","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Read pre-sorted meta data ","metadata":{"papermill":{"duration":0.010414,"end_time":"2022-09-01T07:12:43.432479","exception":false,"start_time":"2022-09-01T07:12:43.422065","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nmulti_ym = pd.read_csv(META_MULTI_TRAIN)","metadata":{"papermill":{"duration":0.111781,"end_time":"2022-09-01T07:12:43.554714","exception":false,"start_time":"2022-09-01T07:12:43.442933","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:04:17.299243Z","iopub.execute_input":"2022-09-06T01:04:17.299833Z","iopub.status.idle":"2022-09-06T01:04:17.430643Z","shell.execute_reply.started":"2022-09-06T01:04:17.299796Z","shell.execute_reply":"2022-09-06T01:04:17.429798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read pre-converted sparse data</br>\nIt is faster and memory effective.","metadata":{"papermill":{"duration":0.010265,"end_time":"2022-09-01T07:12:43.575633","exception":false,"start_time":"2022-09-01T07:12:43.565368","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nmulti_y = load_npz(SPS_MULTIOME_TRAIN_TARGETS)","metadata":{"papermill":{"duration":25.00483,"end_time":"2022-09-01T07:13:08.590955","exception":false,"start_time":"2022-09-01T07:12:43.586125","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:04:17.432041Z","iopub.execute_input":"2022-09-06T01:04:17.432619Z","iopub.status.idle":"2022-09-06T01:04:42.894244Z","shell.execute_reply.started":"2022-09-06T01:04:17.432584Z","shell.execute_reply":"2022-09-06T01:04:42.893181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cross validation","metadata":{"papermill":{"duration":0.01064,"end_time":"2022-09-01T07:13:08.612548","exception":false,"start_time":"2022-09-01T07:13:08.601908","status":"completed"},"tags":[]}},{"cell_type":"code","source":"kf = KFold(n_splits=5,shuffle=True,random_state=1)\nscores = np.zeros((10,2))\nfor consider_day in [True,False]:\n    print(\"Day factor considered.\" if consider_day else \"Day factor ignored.\")\n    for fold,(idx_tr,idx_va) in enumerate(kf.split(multi_y)):\n        kf_tr_y,kf_tr_ym = multi_y[idx_tr],multi_ym.iloc[idx_tr]\n        kf_multi_means = {}\n        kf_tr_ym.head()\n        kf_multi_all_means = kf_tr_y.mean(axis=0)\n        for d in kf_tr_ym.day.unique():\n            kf_multi_means[d] = kf_tr_y[kf_tr_ym.day==d,:].mean(axis=0)\n        del kf_tr_y,kf_tr_ym\n        gc.collect()\n        kf_va_y,kf_va_ym = multi_y[idx_va],multi_ym.iloc[idx_va]\n        kf_va_ypred = np.zeros(kf_va_y.shape)\n        for day in kf_va_ym.day.unique():\n            kf_va_ypred[kf_va_ym.day==day] = (kf_multi_means[day] if consider_day else kf_multi_all_means)\n        kf_va_y = kf_va_y.toarray()\n        mse = mean_squared_error(kf_va_y,kf_va_ypred)\n        corrscore = correlation_score(kf_va_y,kf_va_ypred)\n        print(f\"Fold {fold}:mse = {mse:.5f},corr = {corrscore:.3f}\")\n        del kf_va_y,kf_va_ym,kf_va_ypred\n        gc.collect()\n        scores[fold + 5*(1 if consider_day else 0) ,:] = [mse,corrscore]\n        ","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:04:42.895387Z","iopub.execute_input":"2022-09-06T01:04:42.895694Z","iopub.status.idle":"2022-09-06T01:08:41.243328Z","shell.execute_reply.started":"2022-09-06T01:04:42.895666Z","shell.execute_reply":"2022-09-06T01:08:41.242002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_av = scores[0:5,:].mean(axis=0)\nday_av = scores[5:10,:].mean(axis=0)\nprint(\"All Average : mse=%f correlation=%f\"%(all_av[0],all_av[1]))\nprint(\"day Average : mse=%f correlation=%f\"%(day_av[0],day_av[1]))\nprint(\"Improvement : mse=%f correlation=%f\"%(day_av[0]-all_av[0],day_av[1]-all_av[1]))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:08:41.245162Z","iopub.execute_input":"2022-09-06T01:08:41.245485Z","iopub.status.idle":"2022-09-06T01:08:41.253026Z","shell.execute_reply.started":"2022-09-06T01:08:41.245456Z","shell.execute_reply":"2022-09-06T01:08:41.251604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate means for submission with whole data","metadata":{}},{"cell_type":"code","source":"multi_means = {}\nmulti_ym.head()\nfor d in multi_ym.day.unique():\n    multi_means[d] = multi_y[multi_ym.day==d,:].mean(axis=0)","metadata":{"papermill":{"duration":16.723555,"end_time":"2022-09-01T07:13:25.346907","exception":false,"start_time":"2022-09-01T07:13:08.623352","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:08:41.254943Z","iopub.execute_input":"2022-09-06T01:08:41.255583Z","iopub.status.idle":"2022-09-06T01:08:46.093495Z","shell.execute_reply.started":"2022-09-06T01:08:41.255547Z","shell.execute_reply":"2022-09-06T01:08:46.091527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make day-gene_id DataFrame for later indexing.  ","metadata":{"papermill":{"duration":0.010897,"end_time":"2022-09-01T07:13:25.369285","exception":false,"start_time":"2022-09-01T07:13:25.358388","status":"completed"},"tags":[]}},{"cell_type":"code","source":"multi_means = pd.DataFrame(np.concatenate(([multi_means[c] for c in multi_means.keys() ])),\n                           columns  = pd.read_hdf(FP_MULTIOME_TRAIN_TARGETS, start=0, stop=1).columns,\n                           index = [c for c in multi_means.keys()])\nmulti_means","metadata":{"papermill":{"duration":0.153373,"end_time":"2022-09-01T07:13:25.53372","exception":false,"start_time":"2022-09-01T07:13:25.380347","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:08:46.094991Z","iopub.execute_input":"2022-09-06T01:08:46.095384Z","iopub.status.idle":"2022-09-06T01:08:46.449306Z","shell.execute_reply.started":"2022-09-06T01:08:46.095339Z","shell.execute_reply":"2022-09-06T01:08:46.448352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare for day 10(private test) with liner extrapolation.","metadata":{}},{"cell_type":"code","source":"multi_interp = interpolate.interp1d(np.array([2,3,4,7]).T,\n                                   multi_means,axis=0,fill_value='extrapolate')\nmulti_means.loc[10] = multi_interp(10)\n##multi_means.loc[10] = 0*multi_interp(10)  ## replace 0 to check LB score do not change","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:08:46.450489Z","iopub.execute_input":"2022-09-06T01:08:46.451499Z","iopub.status.idle":"2022-09-06T01:08:46.469253Z","shell.execute_reply.started":"2022-09-06T01:08:46.451463Z","shell.execute_reply":"2022-09-06T01:08:46.468265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_means","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:08:46.470455Z","iopub.execute_input":"2022-09-06T01:08:46.470773Z","iopub.status.idle":"2022-09-06T01:08:46.497657Z","shell.execute_reply.started":"2022-09-06T01:08:46.470744Z","shell.execute_reply":"2022-09-06T01:08:46.496471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Delete variables not needed anymore.","metadata":{}},{"cell_type":"code","source":"del multi_y,multi_ym\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:08:46.499915Z","iopub.execute_input":"2022-09-06T01:08:46.500651Z","iopub.status.idle":"2022-09-06T01:08:46.638335Z","shell.execute_reply.started":"2022-09-06T01:08:46.500605Z","shell.execute_reply":"2022-09-06T01:08:46.636985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make submission data","metadata":{}},{"cell_type":"code","source":"%%time\neval_id = pd.read_csv(FP_EVALUATION_IDS)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:08:46.641134Z","iopub.execute_input":"2022-09-06T01:08:46.642414Z","iopub.status.idle":"2022-09-06T01:09:41.386610Z","shell.execute_reply.started":"2022-09-06T01:08:46.642371Z","shell.execute_reply":"2022-09-06T01:09:41.385401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CITE","metadata":{}},{"cell_type":"code","source":"%%time\ncite_ytm = pd.read_csv(META_CITE_TEST)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:09:41.388501Z","iopub.execute_input":"2022-09-06T01:09:41.388891Z","iopub.status.idle":"2022-09-06T01:09:41.463438Z","shell.execute_reply.started":"2022-09-06T01:09:41.388855Z","shell.execute_reply":"2022-09-06T01:09:41.462199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#eval_id.insert(3,'target',pd.DataFrame(np.zeros((eval_id.shape[0],1))))  ## Overwriting this is very slow, we should use numpy array.\ntgt = np.zeros((eval_id.shape[0],1))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:09:41.465025Z","iopub.execute_input":"2022-09-06T01:09:41.465950Z","iopub.status.idle":"2022-09-06T01:09:41.472217Z","shell.execute_reply.started":"2022-09-06T01:09:41.465903Z","shell.execute_reply":"2022-09-06T01:09:41.471191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor idx in range(0,48663*140,140):\n    cell_id = eval_id.iloc[idx,1]\n    d = cite_ytm[cite_ytm.cell_id==cell_id].day.values[0]\n#    eval_id.iloc[idx:idx+140,3] = cite_means[d].T  ## this is very slow\n    tgt[idx:idx+140,0] = cite_means[d]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:09:41.473867Z","iopub.execute_input":"2022-09-06T01:09:41.474316Z","iopub.status.idle":"2022-09-06T01:12:48.992306Z","shell.execute_reply.started":"2022-09-06T01:09:41.474262Z","shell.execute_reply":"2022-09-06T01:12:48.990551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Multiome","metadata":{}},{"cell_type":"code","source":"%%time\nmulti_ytm = pd.read_csv(META_MULTI_TEST)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T01:12:48.994367Z","iopub.execute_input":"2022-09-06T01:12:48.994877Z","iopub.status.idle":"2022-09-06T01:12:49.087347Z","shell.execute_reply.started":"2022-09-06T01:12:48.994820Z","shell.execute_reply":"2022-09-06T01:12:49.086076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Write cell-gene pairs specified in evaluation_ids.csv </br>\nThis takes shorter time than cite data. It is a surprise to me.","metadata":{"papermill":{"duration":0.010652,"end_time":"2022-09-01T07:13:25.555482","exception":false,"start_time":"2022-09-01T07:13:25.54483","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nmulti_start = 48663*140\nfor idx in range(multi_start,eval_id.shape[0],3512):\n    cell_id = eval_id.iloc[idx,1]\n    d = multi_ytm[multi_ytm.cell_id==cell_id].day.values[0]\n    tgt_gene = eval_id.iloc[idx:idx+3512,2]\n    tgt[idx:idx+3512,0] = multi_means[multi_means.index==d][tgt_gene]\n\n    ","metadata":{"papermill":{"duration":134.286674,"end_time":"2022-09-01T07:15:39.853008","exception":false,"start_time":"2022-09-01T07:13:25.566334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:12:49.089067Z","iopub.execute_input":"2022-09-06T01:12:49.089997Z","iopub.status.idle":"2022-09-06T01:15:05.124517Z","shell.execute_reply.started":"2022-09-06T01:12:49.089961Z","shell.execute_reply":"2022-09-06T01:15:05.123194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Convert the result to DataFrame","metadata":{"papermill":{"duration":0.010889,"end_time":"2022-09-01T07:15:39.874954","exception":false,"start_time":"2022-09-01T07:15:39.864065","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nsubm = pd.DataFrame(tgt,columns = ['target'])\nsubm.index.name='row_id'\nsubm = subm.round(6)\nsubm\n","metadata":{"papermill":{"duration":0.513279,"end_time":"2022-09-01T07:15:40.399269","exception":false,"start_time":"2022-09-01T07:15:39.88599","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:15:05.127379Z","iopub.execute_input":"2022-09-06T01:15:05.128343Z","iopub.status.idle":"2022-09-06T01:15:05.806492Z","shell.execute_reply.started":"2022-09-06T01:15:05.128304Z","shell.execute_reply":"2022-09-06T01:15:05.805368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Write to submission csv file.","metadata":{"papermill":{"duration":0.010862,"end_time":"2022-09-01T07:15:40.421518","exception":false,"start_time":"2022-09-01T07:15:40.410656","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nsubm.to_csv('submission.csv')\n","metadata":{"papermill":{"duration":122.89224,"end_time":"2022-09-01T07:17:43.324797","exception":false,"start_time":"2022-09-01T07:15:40.432557","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:15:05.808055Z","iopub.execute_input":"2022-09-06T01:15:05.808467Z","iopub.status.idle":"2022-09-06T01:17:09.587327Z","shell.execute_reply.started":"2022-09-06T01:15:05.808435Z","shell.execute_reply":"2022-09-06T01:17:09.586216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del tgt,multi_ytm,cite_ytm\ngc.collect()","metadata":{"papermill":{"duration":0.190959,"end_time":"2022-09-01T07:17:43.527334","exception":false,"start_time":"2022-09-01T07:17:43.336375","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-06T01:17:09.588921Z","iopub.execute_input":"2022-09-06T01:17:09.589714Z","iopub.status.idle":"2022-09-06T01:17:09.807838Z","shell.execute_reply.started":"2022-09-06T01:17:09.589669Z","shell.execute_reply":"2022-09-06T01:17:09.806703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}