{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport random\nimport joblib\nimport os\nimport gc\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\nfrom sklearn.preprocessing import OneHotEncoder,LabelEncoder\nfrom sklearn.model_selection import train_test_split,KFold,GroupKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-20T09:30:16.533592Z","iopub.execute_input":"2022-11-20T09:30:16.534122Z","iopub.status.idle":"2022-11-20T09:30:18.214618Z","shell.execute_reply.started":"2022-11-20T09:30:16.533971Z","shell.execute_reply":"2022-11-20T09:30:18.213371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LGBM\nclass MultiOutputLGBMRegressor:\n    def __init__(self,params):\n        self.params = params\n        self.model_list = []\n\n    def fit(self,train_data,train_label,val_data,val_label,**fit_params):\n        output_num = train_label.shape[1]\n        for i in tqdm(range(output_num),leave=False):\n            train_set = lgb.Dataset(train_data,train_label[:,i])\n            val_set = lgb.Dataset(val_data,val_label[:,i])\n            model = lgb.train(\n                self.params,\n                train_set,\n                valid_sets = val_set,\n                callbacks=[\n                    lgb.early_stopping(20,verbose = False),\n                    # lgb.log_evaluation(100),\n                ]\n            )\n            self.model_list.append(model)\n            \n    def predict(self,test_data):\n        res_list = []\n        for model in tqdm(self.model_list,leave=False):\n            res = model.predict(test_data)\n            res_list.append(res)\n        res_list = np.stack(res_list,axis = 1)\n        return res_list\n        \n    def dump(self,path = \"./models/MOLGB/\" ):\n        count = 0\n        os.makedirs(path,exist_ok=True)\n        for model in tqdm(self.model_list,leave=False):\n            joblib.dump(model, f'{path}model_{str(count)}.pkl')\n            count += 1\n        print(\"Model saved\")\n\n    def load(self,path = \"./models/MOLGB/\" ):\n        models = os.listdir(path)\n        if len(self.model_list) != 0:\n            raise ValueError(\"Don't load! Already loaded!\")\n        else:\n            for i in tqdm(range(len(models)),leave=False):\n                # print(\"begin\")\n                model = joblib.load(f'{path}model_{i}.pkl')\n                # print(\"end\")\n                self.model_list.append(model)\n            print(\"Model loaded\")\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:31:00.400659Z","iopub.execute_input":"2022-11-20T09:31:00.401054Z","iopub.status.idle":"2022-11-20T09:31:00.414508Z","shell.execute_reply.started":"2022-11-20T09:31:00.401014Z","shell.execute_reply":"2022-11-20T09:31:00.413209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n  'learning_rate': 0.1, \n  'objective': 'mse', \n  'metric': ['mse', 'mae'], \n  'n_estimators': 10000, \n  'learning_rate': 0.011322411312518462, \n  'num_leaves': 350, \n  'verbose': -1, \n  'boosting_type': 'gbdt', \n  'reg_alpha': 0.40300033428422216, \n  'reg_lambda': 1.6473388122802188, \n  'colsample_bytree': 0.5, \n  'subsample': 0.7, \n  'max_depth': -1, \n  'min_child_samples': 54, \n  'cat_smooth': 41.24648150772993\n}","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:31:10.787731Z","iopub.execute_input":"2022-11-20T09:31:10.788526Z","iopub.status.idle":"2022-11-20T09:31:10.796215Z","shell.execute_reply.started":"2022-11-20T09:31:10.788484Z","shell.execute_reply":"2022-11-20T09:31:10.794855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = np.load(\"../input/cite-final/new_cite_train_final.npz\")[\"arr_0\"]\ntarget = pd.read_hdf(\"../input/open-problems-multimodal/train_cite_targets.h5\").values\ntarget -= target.mean(axis=1).reshape(-1, 1)\ntarget /= target.std(axis=1).reshape(-1, 1)\nprint(train.shape,target.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:30:44.982375Z","iopub.execute_input":"2022-11-20T09:30:44.982813Z","iopub.status.idle":"2022-11-20T09:30:50.406703Z","shell.execute_reply.started":"2022-11-20T09:30:44.982779Z","shell.execute_reply":"2022-11-20T09:30:50.405447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_index = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/train_cite_inputs_idxcol.npz\",allow_pickle=True)\nmeta = pd.read_csv(\"../input/open-problems-multimodal/metadata.csv\",index_col = \"cell_id\")\nmeta = meta[meta.technology==\"citeseq\"]\nlbe = LabelEncoder()\nmeta[\"cell_type\"] = lbe.fit_transform(meta[\"cell_type\"])\nmeta[\"gender\"] = meta.apply(lambda x:0 if x[\"donor\"]==13176 else 1,axis =1)\nmeta_train = meta.reindex(train_index[\"index\"])\n# train_meta = meta_train[\"gender\"].values.reshape(-1, 1)\n# train = np.concatenate([train,train_meta],axis= -1)\ntrain_meta = meta_train[\"cell_type\"].values.reshape(-1, 1)\nohe = OneHotEncoder(sparse=False)\ntrain_meta = ohe.fit_transform(train_meta)\ntrain = np.concatenate([train,train_meta],axis= -1)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:30:50.408945Z","iopub.execute_input":"2022-11-20T09:30:50.410130Z","iopub.status.idle":"2022-11-20T09:30:52.541574Z","shell.execute_reply.started":"2022-11-20T09:30:50.410083Z","shell.execute_reply":"2022-11-20T09:30:52.540464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    if y_true.shape != y_pred.shape: raise ValueError(\"Shapes are different.\")\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:57:42.135999Z","iopub.execute_input":"2022-11-19T12:57:42.136424Z","iopub.status.idle":"2022-11-19T12:57:42.144780Z","shell.execute_reply.started":"2022-11-19T12:57:42.136376Z","shell.execute_reply":"2022-11-19T12:57:42.142881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"np.random.seed(42)\nkf = GroupKFold(n_splits=3) \n\nn = 3\nscores = []\n \nfor id,(idx_tr, idx_va) in enumerate(kf.split(range(train.shape[0]),groups= meta_train.donor)):\n    Xtr, Xva = train[idx_tr], train[idx_va]\n    Ytr, Yva = target[idx_tr], target[idx_va]\n    print(f'Fold {id}..')\n#     random.seed(42)\n#     index_to_train = random.choices([i for i in range(Xtr.shape[0])],k = 1000) \n#     Xtr = Xtr[index_to_train]\n#     Ytr = Ytr[index_to_train]\n    \n    model = MultiOutputLGBMRegressor(params)\n    model.fit(Xtr, Ytr,Xva,Yva,)\n\n    y_tr_pred = model.predict(Xtr)\n    mse_tr = mean_squared_error(Ytr, y_tr_pred)\n    mae_tr = mean_absolute_error(Ytr, y_tr_pred)\n    pearson_tr = correlation_score(Ytr, y_tr_pred)\n    print(f\"Flod_{id}_train  mse:{mse_tr},  mae:{mae_tr},  pearson:{pearson_tr}\")\n\n    y_va_pred = model.predict(Xva)\n    mse = mean_squared_error(Yva, y_va_pred)\n    mae = mean_absolute_error(Yva, y_va_pred)\n    pearson = correlation_score(Yva, y_va_pred)\n    print(f\"Flod-{id}_test   mse:{mse},  mae:{mae},  pearson:{pearson}\\n\")\n    scores.append(pearson)\n    \n    del Xtr, Ytr\n    del Xva, Yva\n    gc.collect()\n    \n    d_path = f\"./models/CV/Fold_{id}/\"\n    os.makedirs(d_path,exist_ok=True)\n    model.dump(d_path)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T13:12:41.556588Z","iopub.execute_input":"2022-11-19T13:12:41.557014Z","iopub.status.idle":"2022-11-19T13:30:36.067107Z","shell.execute_reply.started":"2022-11-19T13:12:41.556978Z","shell.execute_reply":"2022-11-19T13:30:36.065922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2022-11-19T13:30:36.069201Z","iopub.execute_input":"2022-11-19T13:30:36.069607Z","iopub.status.idle":"2022-11-19T13:30:36.076946Z","shell.execute_reply.started":"2022-11-19T13:30:36.069572Z","shell.execute_reply":"2022-11-19T13:30:36.075815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting","metadata":{}},{"cell_type":"code","source":"test = np.load(\"../input/cite-final/new_cite_test_final.npz\")[\"arr_0\"]\ntest_index = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/test_cite_inputs_idxcol.npz\",allow_pickle=True)\nmeta_test = meta.reindex(test_index[\"index\"])\ntest_meta = meta_test[\"cell_type\"].values.reshape(-1, 1)\ntest_meta = ohe.transform(test_meta)\ntest = np.concatenate([test,test_meta],axis= -1)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:30:52.543036Z","iopub.execute_input":"2022-11-20T09:30:52.543489Z","iopub.status.idle":"2022-11-20T09:30:53.117197Z","shell.execute_reply.started":"2022-11-20T09:30:52.543456Z","shell.execute_reply":"2022-11-20T09:30:53.116121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def std(x):\n    return (x - np.mean(x,axis=1).reshape(-1,1)) / np.std(x,axis=1).reshape(-1,1)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:30:53.119375Z","iopub.execute_input":"2022-11-20T09:30:53.119700Z","iopub.status.idle":"2022-11-20T09:30:53.124893Z","shell.execute_reply.started":"2022-11-20T09:30:53.119673Z","shell.execute_reply":"2022-11-20T09:30:53.123743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nimport glob\nmodel_path = \"../input/msci-cite-trained-multioutput-lgbm-models/models/CV/Fold_*\"\nmodel_list = glob.glob(model_path)\npreds = np.zeros((test.shape[0], 140))\nfor id,fn in enumerate(tqdm(model_list)):\n    model_ = MultiOutputLGBMRegressor(params)\n    model_.load(fn+\"/\")\n    preds += std(model_.predict(test))\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:35:35.688836Z","iopub.execute_input":"2022-11-20T09:35:35.689266Z","iopub.status.idle":"2022-11-20T09:46:01.505898Z","shell.execute_reply.started":"2022-11-20T09:35:35.689206Z","shell.execute_reply":"2022-11-20T09:46:01.504715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nsns.heatmap(preds)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:47:33.242781Z","iopub.execute_input":"2022-11-20T09:47:33.243204Z","iopub.status.idle":"2022-11-20T09:47:43.036110Z","shell.execute_reply.started":"2022-11-20T09:47:33.243172Z","shell.execute_reply":"2022-11-20T09:47:43.035280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(test_pred,multi_path ):\n    submission = pd.read_csv(multi_path,index_col = 0)\n    submission = submission[\"target\"]\n    print(\"data loaded\")\n    submission.iloc[:len(test_pred.ravel())] = test_pred.ravel()\n    assert not submission.isna().any()\n    # submission = submission.round(6) # reduce the size of the csv\n    print(\"start -> submission.csv\")\n    submission.to_csv('submission.csv')\n    print(\"submission.csv saved!\")","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:47:43.037808Z","iopub.execute_input":"2022-11-20T09:47:43.038655Z","iopub.status.idle":"2022-11-20T09:47:43.044744Z","shell.execute_reply.started":"2022-11-20T09:47:43.038619Z","shell.execute_reply":"2022-11-20T09:47:43.043576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsubmit(preds,\"../input/4th-solution-ensemble/submission.zip\")","metadata":{"execution":{"iopub.status.busy":"2022-11-20T09:47:43.046520Z","iopub.execute_input":"2022-11-20T09:47:43.046891Z","iopub.status.idle":"2022-11-20T09:51:56.076970Z","shell.execute_reply.started":"2022-11-20T09:47:43.046861Z","shell.execute_reply":"2022-11-20T09:51:56.075796Z"},"trusted":true},"execution_count":null,"outputs":[]}]}