{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"inspired by : https://www.kaggle.com/code/jirkaborovec/mmscel-inst-eda-stat-predictions/notebook?scriptVersionId=103611408","metadata":{}},{"cell_type":"code","source":"! pip install -q tables  # needed for loading HDF files","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-04T19:50:26.798298Z","iopub.execute_input":"2022-09-04T19:50:26.799036Z","iopub.status.idle":"2022-09-04T19:50:41.224870Z","shell.execute_reply.started":"2022-09-04T19:50:26.798908Z","shell.execute_reply":"2022-09-04T19:50:41.223263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\n\nPATH_DATASET = \"/kaggle/input/open-problems-multimodal\"\n\nclass MyDict(dict):\n    def __missing__(self, key):\n        return key","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:50:41.227652Z","iopub.execute_input":"2022-09-04T19:50:41.228100Z","iopub.status.idle":"2022-09-04T19:50:41.237442Z","shell.execute_reply.started":"2022-09-04T19:50:41.228061Z","shell.execute_reply":"2022-09-04T19:50:41.236518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta = pd.read_csv(os.path.join(PATH_DATASET, \"metadata.csv\"))\ndisplay(df_meta.head())\nprint(f\"table size: {len(df_meta)}\")","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:50:41.238440Z","iopub.execute_input":"2022-09-04T19:50:41.238775Z","iopub.status.idle":"2022-09-04T19:50:41.653710Z","shell.execute_reply.started":"2022-09-04T19:50:41.238745Z","shell.execute_reply":"2022-09-04T19:50:41.652466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"donors = list(df_meta.donor.unique())[1::]\ndays = list(df_meta.day.unique())\ncell_typedic = dict(zip(df_meta.cell_type.unique(), range(1,9)))\ncells =list(df_meta.cell_type.unique())[0:-1:]","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:50:41.657635Z","iopub.execute_input":"2022-09-04T19:50:41.658138Z","iopub.status.idle":"2022-09-04T19:50:41.722537Z","shell.execute_reply.started":"2022-09-04T19:50:41.658093Z","shell.execute_reply":"2022-09-04T19:50:41.721499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta['cell_type'] = df_meta['cell_type'].map(cell_typedic)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:50:41.724362Z","iopub.execute_input":"2022-09-04T19:50:41.724718Z","iopub.status.idle":"2022-09-04T19:50:41.770842Z","shell.execute_reply.started":"2022-09-04T19:50:41.724686Z","shell.execute_reply":"2022-09-04T19:50:41.769545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval = pd.read_csv(os.path.join(PATH_DATASET, \"evaluation_ids.csv\"))\ndisplay(df_eval.head())\n      \nprint(f\"total: {len(df_eval)}\")\nprint(f\"cell_id: {len(df_eval['cell_id'].unique())}\")\nprint(f\"gene_id: {len(df_eval['gene_id'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:50:41.772607Z","iopub.execute_input":"2022-09-04T19:50:41.773679Z","iopub.status.idle":"2022-09-04T19:51:53.045321Z","shell.execute_reply.started":"2022-09-04T19:50:41.773633Z","shell.execute_reply":"2022-09-04T19:51:53.043850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval = df_eval.merge(df_meta[['cell_id', 'day', 'donor', 'cell_type', 'technology']], how = 'left', on = 'cell_id').set_index(\"cell_id\")\ndf_meta = df_meta.set_index(\"cell_id\")\ndf_eval['target'] = 0 \ndf_eval","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:51:53.046879Z","iopub.execute_input":"2022-09-04T19:51:53.047255Z","iopub.status.idle":"2022-09-04T19:52:28.124850Z","shell.execute_reply.started":"2022-09-04T19:51:53.047221Z","shell.execute_reply":"2022-09-04T19:52:28.123296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_hdf(os.path.join(PATH_DATASET, \"train_cite_targets.h5\")).astype(np.float16)\ncols_target = list(df.columns)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:28.127093Z","iopub.execute_input":"2022-09-04T19:52:28.127548Z","iopub.status.idle":"2022-09-04T19:52:29.177984Z","shell.execute_reply.started":"2022-09-04T19:52:28.127512Z","shell.execute_reply":"2022-09-04T19:52:29.176570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.join(df_meta[['day','donor','cell_type']], how=\"left\")\nprint(f\"total: {len(df)}\")\nprint(f\"cell_id: {len(df)}\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:29.180305Z","iopub.execute_input":"2022-09-04T19:52:29.181330Z","iopub.status.idle":"2022-09-04T19:52:29.324075Z","shell.execute_reply.started":"2022-09-04T19:52:29.181278Z","shell.execute_reply":"2022-09-04T19:52:29.322813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"protein_pred = pd.DataFrame()\nfor cell in range(1,8):\n    for donor in donors:\n        df_p = df[(df.donor == donor) & (df.cell_type == cell)].groupby(['day']).aggregate('mean').reset_index()\n        if len(df_p) < 3:\n            df_p = df[(df.cell_type == cell)].groupby(['day']).aggregate('mean').reset_index()\n            df_p.donor = donor\n        protein_pred = pd.concat([protein_pred, df_p])\nprotein_pred = protein_pred.reset_index(drop = True)\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:29.329254Z","iopub.execute_input":"2022-09-04T19:52:29.330143Z","iopub.status.idle":"2022-09-04T19:52:29.672739Z","shell.execute_reply.started":"2022-09-04T19:52:29.330101Z","shell.execute_reply":"2022-09-04T19:52:29.671277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cell in range(1,8):\n    for day in days:\n        df_p = df[(df.day == day) & (df.cell_type == cell)].groupby(['day']).aggregate('mean').reset_index()\n        df_p.cell_type = cell\n        df_p.donor = 27678 # o faltante\n        protein_pred = pd.concat([protein_pred, df_p])\nprotein_pred = protein_pred.reset_index(drop = True)\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:29.674449Z","iopub.execute_input":"2022-09-04T19:52:29.674879Z","iopub.status.idle":"2022-09-04T19:52:30.104575Z","shell.execute_reply.started":"2022-09-04T19:52:29.674838Z","shell.execute_reply":"2022-09-04T19:52:30.102991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for donor in donors:\n    for day in days:\n        df_p = df[(df.day == day) & (df.donor == donor)].groupby(['day']).aggregate('mean').reset_index()\n        df_p.cell_type = 8 #interesse faltante\n        df_p.donor = donor\n        protein_pred = pd.concat([protein_pred, df_p])\nprotein_pred = protein_pred.reset_index(drop = True)\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:30.106281Z","iopub.execute_input":"2022-09-04T19:52:30.106967Z","iopub.status.idle":"2022-09-04T19:52:30.430275Z","shell.execute_reply.started":"2022-09-04T19:52:30.106930Z","shell.execute_reply":"2022-09-04T19:52:30.428905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for day in days:\n    df_p = df[(df.day == day)].groupby(['day']).aggregate('mean').reset_index()\n    df_p.cell_type = 8 #interesse faltante\n    df_p.donor = 27678 #interessante faltante\n    protein_pred = pd.concat([protein_pred, df_p])\nprotein_pred = protein_pred.reset_index(drop = True)\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:30.431974Z","iopub.execute_input":"2022-09-04T19:52:30.432739Z","iopub.status.idle":"2022-09-04T19:52:30.713778Z","shell.execute_reply.started":"2022-09-04T19:52:30.432692Z","shell.execute_reply":"2022-09-04T19:52:30.712397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"up_donors = [27678] + donors\nfor cell in range(1,9):\n    for donor in up_donors:\n        lista = [7]\n        x = protein_pred.loc[(protein_pred.donor == donor) & (protein_pred.cell_type == cell)][['day']]\n        x = np.array([valor[0] for valor in x.values])\n        for feature in protein_pred.drop(columns = ['day', 'donor', 'cell_type']).columns:\n            y = protein_pred.loc[(protein_pred.donor == donor) & (protein_pred.cell_type == cell)][[f'{feature}']].mean()[0]\n            #y = np.array([valor[0] for valor in y.values])\n            #modelo_7 = np.poly1d(np.polyfit(x.astype('float64') ,y.astype('float64') , 2))\n            #lista.append(modelo_7(np.array([7]))[0])\n            lista.append(y)\n\n        lista.append(donor)\n        lista.append(cell)\n        df_novo = pd.DataFrame([lista], columns = protein_pred.columns)\n        protein_pred = pd.concat([protein_pred, df_novo])  \nprotein_pred = protein_pred.reset_index(drop = True)\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:30.715947Z","iopub.execute_input":"2022-09-04T19:52:30.716389Z","iopub.status.idle":"2022-09-04T19:52:40.755633Z","shell.execute_reply.started":"2022-09-04T19:52:30.716340Z","shell.execute_reply":"2022-09-04T19:52:40.754221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"protein_pred['technology'] = 'citeseq'\nprotein_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.766587Z","iopub.status.idle":"2022-09-04T19:52:40.767079Z","shell.execute_reply.started":"2022-09-04T19:52:40.766862Z","shell.execute_reply":"2022-09-04T19:52:40.766884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_final = pd.DataFrame()\nfor cell in range(1,8):\n    for donor in donors:\n        df_temp = pd.DataFrame()\n        for day in [2,3,4,7]:\n            col_sums_2 = []\n            count_2 = 0  \n            for i in range(11):\n                path_h5 = os.path.join(PATH_DATASET, \"train_multi_targets.h5\")\n                df = pd.read_hdf(path_h5, start=i * 10000, stop=(i+1) * 10000)\n                df = df.join(df_meta[['day','donor','cell_type']][(df_meta.day == day) & (df_meta.donor == donor) & (df_meta.cell_type == cell)], how=\"inner\")\n                count_2 += len(df)\n                col_sums_2.append(dict(df.sum()))\n\n            df_multi_ = pd.DataFrame(col_sums_2)\n\n            df = pd.DataFrame((df_multi_.sum() / count_2)).T\n            df_temp = pd.concat([df_temp, df])\n        df_final = pd.concat([df_final, df_temp])\ndf_final = df_final.reset_index(drop = True)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.770573Z","iopub.status.idle":"2022-09-04T19:52:40.771033Z","shell.execute_reply.started":"2022-09-04T19:52:40.770815Z","shell.execute_reply":"2022-09-04T19:52:40.770835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor cell in range(1,8):\n    for day in [2,3,4,7]:\n        col_sums_2 = []\n        count_2 = 0  \n        for i in range(11):\n            path_h5 = os.path.join(PATH_DATASET, \"train_multi_targets.h5\")\n            df = pd.read_hdf(path_h5, start=i * 10000, stop=(i+1) * 10000)\n            df = df.join(df_meta[['day', 'donor','cell_type']][(df_meta.day == day) & (df_meta.cell_type == cell)], how=\"inner\")\n            count_2 += len(df)\n            col_sums_2.append(dict(df.sum()))\n\n        df_multi_ = pd.DataFrame(col_sums_2)\n        df = pd.DataFrame((df_multi_.sum() / count_2)).T\n        df.cell_type = cell\n        df.donor = 27678 # o faltante\n        df_final = pd.concat([df_final, df])\ndf_final = df_final.reset_index(drop = True)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.777426Z","iopub.status.idle":"2022-09-04T19:52:40.778287Z","shell.execute_reply.started":"2022-09-04T19:52:40.778040Z","shell.execute_reply":"2022-09-04T19:52:40.778065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor donor in donors:\n    for day in [2,3,4,7]:\n        col_sums_2 = []\n        count_2 = 0  \n        for i in range(11):\n            path_h5 = os.path.join(PATH_DATASET, \"train_multi_targets.h5\")\n            df = pd.read_hdf(path_h5, start=i * 10000, stop=(i+1) * 10000)\n            df = df.join(df_meta[['day', 'donor','cell_type']][(df_meta.day == day) & (df_meta.donor == donor)], how=\"inner\")\n            count_2 += len(df)\n            col_sums_2.append(dict(df.sum()))\n\n        df_multi_ = pd.DataFrame(col_sums_2)\n        df = pd.DataFrame((df_multi_.sum() / count_2)).T\n        df.cell_type = 8 #interesse faltante\n        df.donor = donor\n        df_final = pd.concat([df_final, df])\ndf_final = df_final.reset_index(drop = True)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.779549Z","iopub.status.idle":"2022-09-04T19:52:40.780261Z","shell.execute_reply.started":"2022-09-04T19:52:40.780031Z","shell.execute_reply":"2022-09-04T19:52:40.780053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor day in [2,3,4,7]:\n    col_sums_2 = []\n    count_2 = 0  \n    for i in range(11):\n        path_h5 = os.path.join(PATH_DATASET, \"train_multi_targets.h5\")\n        df = pd.read_hdf(path_h5, start=i * 10000, stop=(i+1) * 10000)\n        df = df.join(df_meta[['day', 'donor','cell_type']][(df_meta.day == day)], how=\"inner\")\n        count_2 += len(df)\n        col_sums_2.append(dict(df.sum()))\n\n    df_multi_ = pd.DataFrame(col_sums_2)\n    df = pd.DataFrame((df_multi_.sum() / count_2)).T\n    df.cell_type = 8 #interesse faltante\n    df.donor = 27678 #interesse faltante\n    df_final = pd.concat([df_final, df])\ndf_final = df_final.reset_index(drop = True)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.781490Z","iopub.status.idle":"2022-09-04T19:52:40.782223Z","shell.execute_reply.started":"2022-09-04T19:52:40.782001Z","shell.execute_reply":"2022-09-04T19:52:40.782023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor cell in range(1,9):\n    for donor in up_donors:\n        lista = []\n        x = df_final.loc[(df_final.donor == donor) & (df_final.cell_type == cell)][['day']]\n        x = np.array([valor[0] for valor in x.values])\n        for feature in df_final.drop(columns = ['day', 'donor', 'cell_type']).columns:\n            y = df_final.loc[(df_final.donor == donor) & (df_final.cell_type == cell)][[f'{feature}']].mean()[0]\n            #y = np.array([valor[0] for valor in y.values])\n            #modelo_10 = np.poly1d(np.polyfit(x.astype('float64') ,y.astype('float64') , 3))\n            #result = modelo_10(np.array([10]))[0]\n            #if result >=1:\n            #    lista.append(np.log(result))\n            #else:\n            #    lista.append(0)\n            lista.append(y)\n                \n        lista.append(10)\n        lista.append(donor)\n        lista.append(cell)\n        df_novo = pd.DataFrame([lista], columns = df_final.columns)\n        df_final = pd.concat([df_final, df_novo])\ndf_final = df_final.reset_index(drop = True)\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.783488Z","iopub.status.idle":"2022-09-04T19:52:40.784114Z","shell.execute_reply.started":"2022-09-04T19:52:40.783913Z","shell.execute_reply":"2022-09-04T19:52:40.783935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final[df_final < 0] = 0","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.785374Z","iopub.status.idle":"2022-09-04T19:52:40.785759Z","shell.execute_reply.started":"2022-09-04T19:52:40.785567Z","shell.execute_reply":"2022-09-04T19:52:40.785584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final['technology'] = 'multiome'\ndf_final","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.787119Z","iopub.status.idle":"2022-09-04T19:52:40.787806Z","shell.execute_reply.started":"2022-09-04T19:52:40.787581Z","shell.execute_reply":"2022-09-04T19:52:40.787602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor cell in range(1,9):\n    for donor in up_donors:\n        for dia in [2, 3, 4, 7, 10]:\n            dic = dict()\n            dic.update(dict(df_final[(df_final.day == dia) & (df_final.donor == donor) & (df_final.cell_type == cell) & (df_final.technology == 'multiome')].drop(columns = ['day', 'donor', 'technology']).mean()))\n            col_day = MyDict(dic)\n            df_eval['target'].loc[(df_eval.day == dia) & (df_eval.donor == donor) & (df_eval.cell_type == cell) & (df_eval.technology == 'multiome')] = df_eval.loc[(df_eval.day == dia) & (df_eval.donor == donor) & (df_eval.cell_type == cell) & (df_eval.technology == 'multiome')]['gene_id'].map(col_day)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.789008Z","iopub.status.idle":"2022-09-04T19:52:40.789710Z","shell.execute_reply.started":"2022-09-04T19:52:40.789498Z","shell.execute_reply":"2022-09-04T19:52:40.789519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.790938Z","iopub.status.idle":"2022-09-04T19:52:40.791502Z","shell.execute_reply.started":"2022-09-04T19:52:40.791315Z","shell.execute_reply":"2022-09-04T19:52:40.791335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor cell in range(1,9):\n    for donor in up_donors:\n        for dia in [2, 3, 4, 7]:\n            dic = dict()\n            dic.update(dict(protein_pred[(protein_pred.day == dia) & (protein_pred.donor == donor) & (protein_pred.cell_type == cell) & (protein_pred.technology == 'citeseq')].drop(columns = ['day', 'donor', 'technology']).mean()))\n            col_day = MyDict(dic)\n            df_eval['target'].loc[(df_eval.day == dia) & (df_eval.donor == donor) & (df_eval.cell_type == cell) & (df_eval.technology == 'citeseq')] = df_eval.loc[(df_eval.day == dia) & (df_eval.donor == donor) & (df_eval.cell_type == cell) & (df_eval.technology == 'citeseq')]['gene_id'].map(col_day)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.792774Z","iopub.status.idle":"2022-09-04T19:52:40.793198Z","shell.execute_reply.started":"2022-09-04T19:52:40.792983Z","shell.execute_reply":"2022-09-04T19:52:40.793003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.794508Z","iopub.status.idle":"2022-09-04T19:52:40.794905Z","shell.execute_reply.started":"2022-09-04T19:52:40.794713Z","shell.execute_reply":"2022-09-04T19:52:40.794732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval = df_eval.set_index('row_id')\n\nprint(f\"total: {len(df_eval)}\")\nprint(f\"gene_id: {len(df_eval['gene_id'].unique())}\")\ndf_eval[[\"target\"]].round(6).to_csv(\"submission.csv\")\n\n! ls -lh .\n! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-09-04T19:52:40.796893Z","iopub.status.idle":"2022-09-04T19:52:40.797862Z","shell.execute_reply.started":"2022-09-04T19:52:40.797594Z","shell.execute_reply":"2022-09-04T19:52:40.797630Z"},"trusted":true},"execution_count":null,"outputs":[]}]}