{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Replaceing hdf to sparse npz & separeted metadata, we can reduce loading time more than half.","metadata":{"_cell_guid":"d63c7659-7590-464b-96b2-5af813c7c88b","_uuid":"718bead1-6531-468e-a08e-688e45111390","papermill":{"duration":0.011037,"end_time":"2022-09-13T04:52:22.511036","exception":false,"start_time":"2022-09-13T04:52:22.499999","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Summary\nConvereting hdf to scipy sparse format makes file size smaller and loading time shorter. But it loses cell_id information. Combining sparse format file and corresponding meta data, we can make the loading time 2 times or more faster.<br>\nData set is available at [Open Problems - Multimodal Single-Cell Metadata](https://www.kaggle.com/datasets/konomuabe/open-problems-multimodal-singlecell-metadata)<br>\nFor cite_test_inputs.h5 case, hdf takes about one minite to load, while loading npz and metadata and converting them into DataFrame takes 25 sec. <br><br>\ndf = pd.DataFrame(load_npz(SPS_CITE_TRAIN_INPUTS).toarray(),<br>\n        ,index = pd.read_csv(\"train_cite_meta.csv\").cell_id<br>\n        ,columns = pd.read_hdf(FP_CITE_TRAIN_INPUTS,start=0,stop=1).columns)<br>\n <br><br>\ntrain_cite_inputs.h5  ->  train_cite_inputs.npz + train_cite_meta.csv <br>\ntrain_cite_targets.h5 ->  train_cite_targets.npz + train_cite_meta.csv <br>\ntest_cite_targets.h5  ->  test_cite_targets.npz + test_cite_meta.csv <br>\n\ntrain_multi_inputs.h5  ->  train_multi_inputs.npz + train_multi_meta.csv <br>\ntrain_multi_targets.h5 ->  train_multi_targets.npz + train_multi_meta.csv <br>\ntest_multi_targets.h5  ->  test_multi_targets.npz + test_multi_meta.csv <br>\n <br>\n <br>\nThis also create multiome_submission inputs. Because only parts of test_multi_targets.h5 are used for submission, we have to select the specified data every time. <br>\nFeeding eval_multi_inputs.npz (+ eval_multi_inputs.csv) to your model returns the selected cell_id target.<br>\nYou can arrange the specified gene_ids into submission data in one minutes. Please refer [Comparing submission file creation method](https://www.kaggle.com/code/konomuabe/comparing-submission-file-creation-method)\n\n\n### Topics\nWhen creating metadata file in this note book, changing read_hdf block size from 100 to 5000, and converting evaluation_ids.csv's cell_id to **CategoricalDtype**, metadata creation process time **decreased from** nearly **2 hours to 10 minutes**.","metadata":{"papermill":{"duration":0.011307,"end_time":"2022-09-13T04:52:22.531551","exception":false,"start_time":"2022-09-13T04:52:22.520244","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# installs\n!pip install --quiet tables\n\n# imports\nimport os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport gc\nimport tqdm\nimport h5py\nfrom scipy.sparse import *\nimport psutil\n\n# set paths\nDATA_DIR = \"../input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")","metadata":{"_cell_guid":"fa228cb5-13d8-4cbb-89eb-b1a5060fce84","_kg_hide-input":true,"_uuid":"230d5a88-745a-4833-8d6c-355f3879b466","papermill":{"duration":16.547871,"end_time":"2022-09-13T04:52:39.091228","exception":false,"start_time":"2022-09-13T04:52:22.543357","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-09-25T05:46:09.238469Z","iopub.execute_input":"2022-09-25T05:46:09.239325Z","iopub.status.idle":"2022-09-25T05:46:26.215634Z","shell.execute_reply.started":"2022-09-25T05:46:09.239199Z","shell.execute_reply":"2022-09-25T05:46:26.214487Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the dimension of hdf data","metadata":{"_cell_guid":"64203ec6-dd63-4adf-82de-dd78e536a3df","_kg_hide-input":true,"_uuid":"6d14f2de-9674-46cc-ba4b-a0bec9ee7321","papermill":{"duration":0.0084,"end_time":"2022-09-13T04:52:39.108903","exception":false,"start_time":"2022-09-13T04:52:39.100503","status":"completed"},"tags":[]}},{"cell_type":"code","source":"_locals = locals().copy()\nfor k in _locals.keys():\n    if type(k) is str and k[:3]=='FP_' and eval(k)[-3:]=='.h5':\n        print(k,eval(k))\n        with h5py.File(eval(k),'r') as f:\n            for k1 in f:\n                print(k1)\n                for k2 in f[k1]:\n                    print(f[k1][k2])\n        print(\"\")","metadata":{"_cell_guid":"a7ff6fec-9cc8-4cec-9098-f730f6d1f733","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"7cc14182-4169-43ab-b41d-d10b0bfd8a54","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.114771,"end_time":"2022-09-13T04:52:39.232260","exception":false,"start_time":"2022-09-13T04:52:39.117489","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:46:26.217668Z","iopub.execute_input":"2022-09-25T05:46:26.218278Z","iopub.status.idle":"2022-09-25T05:46:26.333378Z","shell.execute_reply.started":"2022-09-25T05:46:26.218240Z","shell.execute_reply":"2022-09-25T05:46:26.332282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make four metadata files. \nEach have the same cell_id orders with the correspoiding hdf file","metadata":{"_cell_guid":"eb350783-592a-4a3c-bb0c-56670304074e","_uuid":"cee9e794-69a0-470f-b389-11d73407f4a6","papermill":{"duration":0.008368,"end_time":"2022-09-13T04:52:39.249400","exception":false,"start_time":"2022-09-13T04:52:39.241032","status":"completed"},"tags":[]}},{"cell_type":"code","source":"files = {FP_CITE_TRAIN_TARGETS:'cite_train_meta.csv',\n           FP_CITE_TEST_INPUTS: 'cite_test_meta.csv',\n         FP_MULTIOME_TRAIN_TARGETS:'multiome_train_meta.csv',\n         FP_MULTIOME_TEST_INPUTS:'multiome_test_meta.csv',\n        }\n\nhdf_rows = {FP_CITE_TRAIN_TARGETS:70988,\n           FP_CITE_TEST_INPUTS: 48663,\n         FP_MULTIOME_TRAIN_TARGETS:105942,\n         FP_MULTIOME_TEST_INPUTS:55935,\n        }\n\ndf_cell = pd.read_csv(FP_CELL_METADATA)\n","metadata":{"_cell_guid":"01b1a660-db7d-4473-8c41-e7c5b1e43f12","_uuid":"07afbfdc-fac2-4686-a9ba-55e6665f8ceb","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.385774,"end_time":"2022-09-13T04:52:39.643876","exception":false,"start_time":"2022-09-13T04:52:39.258102","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:46:26.335978Z","iopub.execute_input":"2022-09-25T05:46:26.337555Z","iopub.status.idle":"2022-09-25T05:46:26.764135Z","shell.execute_reply.started":"2022-09-25T05:46:26.337503Z","shell.execute_reply":"2022-09-25T05:46:26.762865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Changing **df_cell.cell_id to CategoricalDtype reduces process time 4046 sec.**","metadata":{"papermill":{"duration":0.008444,"end_time":"2022-09-13T04:52:39.661075","exception":false,"start_time":"2022-09-13T04:52:39.652631","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time \ndf_cell.cell_id = df_cell.cell_id.astype(pd.CategoricalDtype())","metadata":{"papermill":{"duration":0.650927,"end_time":"2022-09-13T04:52:40.320663","exception":false,"start_time":"2022-09-13T04:52:39.669736","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:46:26.767224Z","iopub.execute_input":"2022-09-25T05:46:26.767597Z","iopub.status.idle":"2022-09-25T05:46:27.453592Z","shell.execute_reply.started":"2022-09-25T05:46:26.767562Z","shell.execute_reply":"2022-09-25T05:46:27.452594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reading the whole hdf file causes memory error. So split and read the data.","metadata":{"_cell_guid":"37c0ae0d-41ff-43df-9ef0-3fa6921cdf9e","_uuid":"a0c4ef32-1604-43ec-9a76-deb4ad531e11","papermill":{"duration":0.008578,"end_time":"2022-09-13T04:52:40.337943","exception":false,"start_time":"2022-09-13T04:52:40.329365","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Decreasing block_size from 5000 to 100 increases process time 856 sec.**\nBe careful to divide too much.","metadata":{"papermill":{"duration":0.008423,"end_time":"2022-09-13T04:52:40.355424","exception":false,"start_time":"2022-09-13T04:52:40.347001","status":"completed"},"tags":[]}},{"cell_type":"code","source":"block_size = 5000\n#block_size = 100","metadata":{"papermill":{"duration":0.016694,"end_time":"2022-09-13T04:52:40.380864","exception":false,"start_time":"2022-09-13T04:52:40.364170","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:46:27.454682Z","iopub.execute_input":"2022-09-25T05:46:27.455281Z","iopub.status.idle":"2022-09-25T05:46:27.459096Z","shell.execute_reply.started":"2022-09-25T05:46:27.455243Z","shell.execute_reply":"2022-09-25T05:46:27.458224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor file in files.keys():\n    print(file)\n    lines = [block_size]*(hdf_rows[file]//block_size)\n    if hdf_rows[file]%block_size > 0:\n        lines.append(hdf_rows[file]%block_size)\n    start = 0\n    with open(files[file],\"wt\") as f:\n        f.write(\"cell_id,day,donor,cell_type,technology\\n\")\n        for l in tqdm.tqdm(lines):\n            df = pd.read_hdf(file ,start = start,stop = start + l)\n            start += l\n            for id in df.index:\n                r = df_cell[df_cell.cell_id==id].values[0]\n                f.write(\"%s,%d,%s,%s,%s\\n\"%(r[0],r[1],r[2],r[3],r[4]))\n            del df\n            gc.collect()","metadata":{"_cell_guid":"c81b376f-a94f-4ac1-b877-948a6b81ff42","_uuid":"2a12f40a-678f-4241-a132-13337ed3c93a","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":592.233804,"end_time":"2022-09-13T05:02:32.623460","exception":false,"start_time":"2022-09-13T04:52:40.389656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:46:27.460252Z","iopub.execute_input":"2022-09-25T05:46:27.461183Z","iopub.status.idle":"2022-09-25T05:56:04.302418Z","shell.execute_reply.started":"2022-09-25T05:46:27.461145Z","shell.execute_reply":"2022-09-25T05:56:04.301201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fix name inconsistency while keeping backward compatibility.","metadata":{"papermill":{"duration":0.013067,"end_time":"2022-09-13T05:02:32.649957","exception":false,"start_time":"2022-09-13T05:02:32.636890","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!cp cite_train_meta.csv train_cite_meta.csv\n!cp cite_test_meta.csv test_cite.meta.csv\n!cp multiome_train_meta.csv train_multi_meta.csv\n!cp multiome_test_meta.csv test_multi_meta.csv\n","metadata":{"papermill":{"duration":4.538849,"end_time":"2022-09-13T05:02:37.201964","exception":false,"start_time":"2022-09-13T05:02:32.663115","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:56:04.305123Z","iopub.execute_input":"2022-09-25T05:56:04.305489Z","iopub.status.idle":"2022-09-25T05:56:08.740664Z","shell.execute_reply.started":"2022-09-25T05:56:04.305450Z","shell.execute_reply":"2022-09-25T05:56:08.739286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generation of sparse matrix file\nThis takes about an hour.","metadata":{"papermill":{"duration":0.013011,"end_time":"2022-09-13T05:02:37.228363","exception":false,"start_time":"2022-09-13T05:02:37.215352","status":"completed"},"tags":[]}},{"cell_type":"code","source":"hdf_files = [\n    FP_CITE_TRAIN_INPUTS,\n    FP_CITE_TRAIN_TARGETS,\n    FP_CITE_TEST_INPUTS,\n    FP_MULTIOME_TRAIN_INPUTS,\n    FP_MULTIOME_TRAIN_TARGETS,\n    FP_MULTIOME_TEST_INPUTS,\n ]\n\nchunksize = 5000\nfor file in hdf_files:\n    print(file)\n    start = 0\n    total_rows = 0\n    while True:\n        print(start)\n        df = pd.read_hdf(file ,start = start ,stop = start+chunksize)\n        rows_read = len(df)\n        if start == 0 :\n            csr = csr_matrix(df.values)\n        else:\n            csr = vstack([csr,csr_matrix(df.values)])\n        if rows_read<chunksize:\n            break\n        start += chunksize\n        del df\n        gc.collect()           \n    save_file = file.split('/')[-1][:-2]+\"npz\"\n    print(\"saving %s\"%save_file)\n    save_npz(save_file,csr)\n\ntrain_cite_x = load_npz('train_cite_inputs.npz')\ncite_nzclm = (train_cite_x.getnnz(axis=0)!=0)  \ndel train_cite_x\ngc.collect()\nnp.save('cite_columns',cite_nzclm)  \n#train_multi_x = train_multi_x[:,multi_nzclm] ## to extract Non-zero columns\n#train_multi_x = train_multi_x.tocsr()\n\ntrain_multi_x = load_npz('train_multi_inputs.npz')\nmulti_nzclm = (train_multi_x.getnnz(axis=0)!=0)  \ndel train_multi_x\ngc.collect()\n\nnp.save('multi_columns',multi_nzclm)\n","metadata":{"papermill":{"duration":3575.9391,"end_time":"2022-09-13T06:02:13.180687","exception":false,"start_time":"2022-09-13T05:02:37.241587","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-25T05:56:08.743225Z","iopub.execute_input":"2022-09-25T05:56:08.743856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert evaluation_ids.csv to parquet","metadata":{"papermill":{"duration":0.018795,"end_time":"2022-09-13T06:02:13.220495","exception":false,"start_time":"2022-09-13T06:02:13.201700","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Reading parquet format takes only 1/3 times of reading evaluation_ids.csv","metadata":{"papermill":{"duration":0.019016,"end_time":"2022-09-13T06:02:13.258615","exception":false,"start_time":"2022-09-13T06:02:13.239599","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\neval_ids = pd.read_csv(FP_EVALUATION_IDS,index_col='cell_id')\n","metadata":{"papermill":{"duration":55.700976,"end_time":"2022-09-13T06:03:08.978655","exception":false,"start_time":"2022-09-13T06:02:13.277679","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_ids.drop(['row_id'],axis=1)\neval_ids.to_parquet('evaluation_ids.parquet')\ndel eval_ids\ngc.collect()","metadata":{"papermill":{"duration":21.091132,"end_time":"2022-09-13T06:03:30.089579","exception":false,"start_time":"2022-09-13T06:03:08.998447","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\neval_ids = pd.read_parquet('evaluation_ids.parquet')","metadata":{"papermill":{"duration":11.946975,"end_time":"2022-09-13T06:03:42.221061","exception":false,"start_time":"2022-09-13T06:03:30.274086","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Multiome evaluation data","metadata":{"papermill":{"duration":0.019733,"end_time":"2022-09-13T06:03:42.261094","exception":false,"start_time":"2022-09-13T06:03:42.241361","status":"completed"},"tags":[]}},{"cell_type":"code","source":"multi_start = 48663*140\nmulti_eval_cells = eval_ids.index[multi_start::3512]\nmulti_eval_cells\ndel eval_ids\ngc.collect()\nprint(\"1. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))","metadata":{"papermill":{"duration":0.471527,"end_time":"2022-09-13T06:03:42.752329","exception":false,"start_time":"2022-09-13T06:03:42.280802","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_test_meta = pd.read_csv('multiome_test_meta.csv',index_col='cell_id')\nmulti_eval_meta = multi_test_meta.reindex(multi_eval_cells)\nmulti_eval_meta\nprint(\"2. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))","metadata":{"papermill":{"duration":0.121227,"end_time":"2022-09-13T06:03:42.904297","exception":false,"start_time":"2022-09-13T06:03:42.783070","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_eval_meta.to_csv(\"eval_multi_meta.csv\")","metadata":{"papermill":{"duration":0.062863,"end_time":"2022-09-13T06:03:42.987222","exception":false,"start_time":"2022-09-13T06:03:42.924359","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"3. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))","metadata":{"papermill":{"duration":0.030349,"end_time":"2022-09-13T06:03:43.038178","exception":false,"start_time":"2022-09-13T06:03:43.007829","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_test_x = load_npz('test_multi_inputs.npz')\nprint(\"4. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))","metadata":{"papermill":{"duration":23.211365,"end_time":"2022-09-13T06:04:06.269985","exception":false,"start_time":"2022-09-13T06:03:43.058620","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#bksz = 1000\nbksz = 100\nbuf = np.zeros((bksz,multi_test_x.shape[1]))\nidx = 0\nlid = 0\n#for id in multi_eval_cells.cell_id:\nfor id in multi_eval_cells:\n    buf[lid,:] = multi_test_x[multi_test_meta.index == id].toarray()\n    idx += 1\n    lid += 1\n    if idx == bksz:\n        multi_test_eval_x = csr_matrix(buf)\n        lid = 0\n        gc.collect()\n    elif lid==bksz:\n        multi_test_eval_x = vstack([multi_test_eval_x,csr_matrix(buf)])\n        lid = 0\n        gc.collect()\nif lid>0:\n    multi_test_eval_x = vstack([multi_test_eval_x,csr_matrix(buf[:lid])]) # bugfix at version 11\nprint(\"5. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))\nsave_npz(\"eval_multi_inputs\",multi_test_eval_x)\ndel multi_test_eval_x,multi_test_x,multi_test_meta,multi_eval_meta\ngc.collect()\nprint(\"6. Used %.1f Gbyte RAM\"%(psutil.virtual_memory()[3]/(1024**3)))","metadata":{"papermill":{"duration":625.9421,"end_time":"2022-09-13T06:14:32.232380","exception":false,"start_time":"2022-09-13T06:04:06.290280","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Speed comparison","metadata":{"papermill":{"duration":0.019995,"end_time":"2022-09-13T06:14:32.272923","exception":false,"start_time":"2022-09-13T06:14:32.252928","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Running these code here reaches memory limit, so I commented out them. So, I wrote the time measurement result instead.","metadata":{"papermill":{"duration":0.020131,"end_time":"2022-09-13T06:14:32.313812","exception":false,"start_time":"2022-09-13T06:14:32.293681","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#%%time\n#df = pd.read_hdf(FP_CITE_TRAIN_INPUTS)\n# CPU times: user 31.1 s, sys: 9.12 s, total: 40.2 s Wall time: 1min 1s\n","metadata":{"papermill":{"duration":0.028716,"end_time":"2022-09-13T06:14:32.362712","exception":false,"start_time":"2022-09-13T06:14:32.333996","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del df\n#gc.collect()","metadata":{"papermill":{"duration":0.028231,"end_time":"2022-09-13T06:14:32.411422","exception":false,"start_time":"2022-09-13T06:14:32.383191","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n#a = load_npz(SPS_CITE_TRAIN_INPUTS)\n#CPU times: user 13.8 s, sys: 783 ms, total: 14.6 s Wall time: 14.6 s","metadata":{"papermill":{"duration":0.029351,"end_time":"2022-09-13T06:14:32.461088","exception":false,"start_time":"2022-09-13T06:14:32.431737","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n#a = a.toarray()\n# CPU times: user 3.33 s, sys: 7.05 s, total: 10.4 s Wall time: 10.4 s","metadata":{"papermill":{"duration":0.033263,"end_time":"2022-09-13T06:14:32.514860","exception":false,"start_time":"2022-09-13T06:14:32.481597","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n#df = pd.DataFrame(a\n#        ,index = pd.read_csv(\"cite_train_meta.csv\").cell_id\n#        ,columns = pd.read_hdf(FP_CITE_TRAIN_INPUTS,start=0,stop=1).columns)\n# CPU times: user 171 ms, sys: 43.7 ms, total: 215 ms Wall time: 318 ms","metadata":{"papermill":{"duration":0.039318,"end_time":"2022-09-13T06:14:32.582758","exception":false,"start_time":"2022-09-13T06:14:32.543440","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Try to replace day2 ","metadata":{"papermill":{"duration":0.025045,"end_time":"2022-09-13T06:14:32.632270","exception":false,"start_time":"2022-09-13T06:14:32.607225","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"day2 donor 27678 was updated. If I can replace them I will, but it does not seem replacable.","metadata":{"papermill":{"duration":0.019899,"end_time":"2022-09-13T06:14:32.672498","exception":false,"start_time":"2022-09-13T06:14:32.652599","status":"completed"},"tags":[]}},{"cell_type":"code","source":"day2_27678 = pd.read_csv('../input/open-problems-multimodal/metadata_cite_day_2_donor_27678.csv')\nday2_27678.shape","metadata":{"papermill":{"duration":0.056878,"end_time":"2022-09-13T06:14:32.749709","exception":false,"start_time":"2022-09-13T06:14:32.692831","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"day2_27678","metadata":{"papermill":{"duration":0.046714,"end_time":"2022-09-13T06:14:32.817522","exception":false,"start_time":"2022-09-13T06:14:32.770808","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_cells = set(day2_27678.cell_id)\n","metadata":{"papermill":{"duration":0.031491,"end_time":"2022-09-13T06:14:32.870491","exception":false,"start_time":"2022-09-13T06:14:32.839000","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_test_meta = pd.read_csv('cite_test_meta.csv')\ncite_test_meta[((cite_test_meta.day==2) & (cite_test_meta.donor==27678))]","metadata":{"papermill":{"duration":0.089731,"end_time":"2022-09-13T06:14:32.981410","exception":false,"start_time":"2022-09-13T06:14:32.891679","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"initial_cells = set(cite_test_meta[((cite_test_meta.day==2) & (cite_test_meta.donor==27678))].cell_id)","metadata":{"papermill":{"duration":0.034341,"end_time":"2022-09-13T06:14:33.038492","exception":false,"start_time":"2022-09-13T06:14:33.004151","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(initial_cells-new_cells)","metadata":{"papermill":{"duration":0.031359,"end_time":"2022-09-13T06:14:33.091119","exception":false,"start_time":"2022-09-13T06:14:33.059760","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no common cell_ids. We can not replace day2 donor 27678 data with new data ","metadata":{"papermill":{"duration":0.021006,"end_time":"2022-09-13T06:14:33.133326","exception":false,"start_time":"2022-09-13T06:14:33.112320","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.021618,"end_time":"2022-09-13T06:14:33.178445","exception":false,"start_time":"2022-09-13T06:14:33.156827","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}