{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.model_selection import train_test_split\nimport gc\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import defaultdict\nfrom IPython.display import FileLink        \nimport pickle as pkl\n\nimport matplotlib.gridspec as gridspec\n\nimport lightgbm as lgmb\nfrom collections import defaultdict","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#54.2DS  Predict CD protein, based only on RNAs EXCLUDING his own RNA    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    \n    y2 = y_pred.copy()\n    y2 -= y2.mean(axis=0);    y2 /= y2.std(axis=0) \n    y1 = y_true.copy(); \n    y1 -= y1.mean(axis=0);    y1 /= y1.std(axis=0) \n        \n    c = (y1*y2).mean().mean()# Correlation for rescaled matrices is just matrix product and average \n        \n    return c","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rna_df = pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")\ncd_df = pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")\nrename_df = pd.read_csv('/kaggle/input/research-project-01-around-multimodal-singlecell/CD_info.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rename_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_cd_rna = {\n    it['names']:f\"{it['ENSG ID']}_{it['names']}\" for it in rename_df.to_dict(\"records\")\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(map_cd_rna)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_cd_rna","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_cds = [\n    \"CD44\",\n    \"CD41\",\n    \"CD36\",\n    \"CD32\",\n    \"CD88\",\n    \"CD48\",\n    \"CD62L\",\n    \"CD49b\",\n    \"CD45RA\",\n    \"CD82\",\n    \"CD38\",\n    \"CD71\",\n    \"CD115\",\n    \"CD11a\",\n    \"CD244\",\n    \"CD45\"    \n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rna_train, rna_test, cd_train, cd_test = train_test_split(\n    rna_df, \n    cd_df, \n    test_size=0.33, \n    random_state=42\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del rna_df\ndel cd_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = rna_train.columns.tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd_test_df = defaultdict(list)\nfor cd_it in target_cds:\n    \n    try:\n        owned_rna_it = map_cd_rna[cd_it]\n        print(cd_it, owned_rna_it)\n        columns = rna_train.columns.tolist()\n\n        owned_rna_it_index = columns.index(owned_rna_it)\n        del columns[owned_rna_it_index]\n\n        model = lgmb.LGBMRegressor(\n            verbose = 1\n        )    \n\n        model.fit(\n            rna_train[columns],\n            cd_train[cd_it]\n        )    \n\n        model_dir_path = f\"/kaggle/working/{cd_it}/lgmb\"    \n        os.makedirs(model_dir_path, exist_ok=True)\n\n        model_path = model_dir_path + \"/model.txt\"\n        model.booster_.save_model(model_path)    \n\n        cd_test_true = cd_test[cd_it].values\n\n        cd_test_pred = model.predict(\n            rna_test[columns]\n        )    \n\n        corr_score = correlation_score(cd_test_true, cd_test_pred)    \n\n        cd_test_df[\"CD\"].append(cd_it)\n        cd_test_df[\"Exclude_rna\"].append(owned_rna_it)\n        cd_test_df[\"Corr\"].append(corr_score)    \n        print(f\"CD: {cd_it}; Corr: {corr_score}\")\n\n        del cd_test_true\n        del cd_test_pred\n        del model\n        gc.collect()\n    except Exception as e:\n        print(e)\n        \n    #break\n    \ncd_test_df = pd.DataFrame(cd_test_df)\ncd_test_df.to_csv(\"/kaggle/working/test_predict_cd_without_own_rna.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}