{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#准备工作\nimport os, gc, pickle\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom colorama import Fore, Back, Style\nfrom matplotlib.ticker import MaxNLocator\nimport scipy.sparse as sps\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, scale\nfrom sklearn.decomposition import PCA\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.linear_model import Ridge, LinearRegression\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.decomposition import TruncatedSVD\n\nDATA_DIR = \"../input/open-problems-msci-multiome-sparse-matrices/\"\nMULTIOME_TRAIN_INPUTS_SPARSE = os.path.join(DATA_DIR,\"train_multiome_input_sparse.npz\")\nMULTIOME_TRAIN_TARGETS_SPARSE = os.path.join(DATA_DIR,\"train_multi_targets_sparse.npz\")\nMULTIOME_TEST_INPUTS_SPARSE = os.path.join(DATA_DIR,\"test_multi_inputs_sparse.npz\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-27T05:43:18.615241Z","iopub.execute_input":"2022-08-27T05:43:18.615769Z","iopub.status.idle":"2022-08-27T05:43:19.290524Z","shell.execute_reply.started":"2022-08-27T05:43:18.615673Z","shell.execute_reply":"2022-08-27T05:43:19.289374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#皮尔森相关系数计算函数\ndef correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    if y_true.shape != y_pred.shape: raise ValueError(\"Shapes are different.\")\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:54:14.70277Z","iopub.execute_input":"2022-08-27T04:54:14.703178Z","iopub.status.idle":"2022-08-27T04:54:14.710403Z","shell.execute_reply.started":"2022-08-27T04:54:14.703139Z","shell.execute_reply":"2022-08-27T04:54:14.709225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#加载multiome全部数据集的sparse matrix形式\ntrain_inputs = sps.load_npz(MULTIOME_TRAIN_INPUTS_SPARSE)\n#train_targets = sps.load_npz(MULTIOME_TRAIN_TARGETS_SPARSE)\n#test_inputs = sps.load_npz(MULTIOME_TEST_INPUTS_SPARSE)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:15:27.192204Z","iopub.execute_input":"2022-08-27T04:15:27.192881Z","iopub.status.idle":"2022-08-27T04:16:12.738256Z","shell.execute_reply.started":"2022-08-27T04:15:27.192825Z","shell.execute_reply":"2022-08-27T04:16:12.737048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tsvd = TruncatedSVD(n_components=2000, random_state=42)\ntrain_inputs_fit = tsvd.fit_transform(train_inputs)\n\nplt.plot(tsvd.explained_variance_ratio_.cumsum())\nplt.title(\"Cumulative explained variance ratio\")\nplt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))\nplt.xlabel('tsvd component')\nplt.ylabel('Cumulative explained variance ratio')\nplt.show()\nprint(train_inputs_fit.shape)\nnp.save(\"./train_inputs_fit_2000\", train_inputs_fit)","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:40:06.911362Z","iopub.execute_input":"2022-08-27T04:40:06.912038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs_fit = np.load(\"../input/celltsvd/train_inputs_fit.npy\")","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:54:19.294054Z","iopub.execute_input":"2022-08-27T04:54:19.295072Z","iopub.status.idle":"2022-08-27T04:54:19.542988Z","shell.execute_reply.started":"2022-08-27T04:54:19.295032Z","shell.execute_reply":"2022-08-27T04:54:19.541664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_x = train_inputs_fit[0:20000]","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:54:21.496535Z","iopub.execute_input":"2022-08-27T04:54:21.497116Z","iopub.status.idle":"2022-08-27T04:54:21.501575Z","shell.execute_reply.started":"2022-08-27T04:54:21.497059Z","shell.execute_reply":"2022-08-27T04:54:21.500557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_inputs_fit\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:54:23.338823Z","iopub.execute_input":"2022-08-27T04:54:23.339921Z","iopub.status.idle":"2022-08-27T04:54:23.444433Z","shell.execute_reply.started":"2022-08-27T04:54:23.339877Z","shell.execute_reply":"2022-08-27T04:54:23.443329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:54:25.511477Z","iopub.execute_input":"2022-08-27T04:54:25.511837Z","iopub.status.idle":"2022-08-27T04:54:37.578402Z","shell.execute_reply.started":"2022-08-27T04:54:25.511803Z","shell.execute_reply":"2022-08-27T04:54:37.577128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_x.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:04:18.671776Z","iopub.execute_input":"2022-08-27T05:04:18.672527Z","iopub.status.idle":"2022-08-27T05:04:18.68252Z","shell.execute_reply.started":"2022-08-27T05:04:18.672475Z","shell.execute_reply":"2022-08-27T05:04:18.681351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_inputs","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:55:01.688493Z","iopub.execute_input":"2022-08-27T04:55:01.689062Z","iopub.status.idle":"2022-08-27T04:55:01.800697Z","shell.execute_reply.started":"2022-08-27T04:55:01.689012Z","shell.execute_reply":"2022-08-27T04:55:01.799159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-26T13:50:26.337272Z","iopub.execute_input":"2022-08-26T13:50:26.338017Z","iopub.status.idle":"2022-08-26T13:50:26.444153Z","shell.execute_reply.started":"2022-08-26T13:50:26.337977Z","shell.execute_reply":"2022-08-26T13:50:26.442914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T13:50:40.818041Z","iopub.execute_input":"2022-08-26T13:50:40.818827Z","iopub.status.idle":"2022-08-26T13:50:40.824035Z","shell.execute_reply.started":"2022-08-26T13:50:40.81878Z","shell.execute_reply":"2022-08-26T13:50:40.822435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_y = pd.read_hdf(\"../input/open-problems-multimodal/train_multi_targets.h5\", start=0, stop=20000)\ny_columns = multi_train_y.columns\nmulti_train_y = multi_train_y.values","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:55:07.280621Z","iopub.execute_input":"2022-08-27T04:55:07.280986Z","iopub.status.idle":"2022-08-27T04:55:20.424073Z","shell.execute_reply.started":"2022-08-27T04:55:07.280951Z","shell.execute_reply":"2022-08-27T04:55:20.423019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_columns\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-27T04:55:50.790939Z","iopub.execute_input":"2022-08-27T04:55:50.791932Z","iopub.status.idle":"2022-08-27T04:55:50.917248Z","shell.execute_reply.started":"2022-08-27T04:55:50.791879Z","shell.execute_reply":"2022-08-27T04:55:50.916056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_y.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:06:55.953099Z","iopub.execute_input":"2022-08-27T05:06:55.95382Z","iopub.status.idle":"2022-08-27T05:06:55.960648Z","shell.execute_reply.started":"2022-08-27T05:06:55.953779Z","shell.execute_reply":"2022-08-27T05:06:55.959624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_x_all = multi_train_x\nmulti_train_x = multi_train_x_all[0:4000]\nmulti_train_y_all = multi_train_y\nmulti_train_y = multi_train_y_all[0:4000]","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:04:11.972744Z","iopub.execute_input":"2022-08-27T05:04:11.973516Z","iopub.status.idle":"2022-08-27T05:04:11.979918Z","shell.execute_reply.started":"2022-08-27T05:04:11.973473Z","shell.execute_reply":"2022-08-27T05:04:11.978741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_train_x = multi_train_x_all[0:6000]\nmulti_train_y = multi_train_y_all[0:6000]","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:06:53.285169Z","iopub.execute_input":"2022-08-27T05:06:53.286234Z","iopub.status.idle":"2022-08-27T05:06:53.291415Z","shell.execute_reply.started":"2022-08-27T05:06:53.28618Z","shell.execute_reply":"2022-08-27T05:06:53.290281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nimport pandas as pd\nimport numpy as np\n\ndef train(net, train_features, train_labels, test_features, test_labels,\n          num_epochs, learning_rate, weight_decay, batch_size):\n    train_ls, test_ls = [], []\n    train_score, test_score = [], []\n\n    train_iter = TensorDataset(train_features, train_labels)\n    optimizer = torch.optim.Adam(net.parameters(),\n                                 lr = learning_rate,\n                                 weight_decay = weight_decay)\n    #optimizer = torch.optim.SGD(net.parameters(),\n    #                            lr = learning_rate,\n    #                            momentum=0.9,\n    #                            weight_decay = weight_decay)\n    for epoch in range(num_epochs):\n        for X, y in tqdm.tqdm(train_iter):\n            optimizer.zero_grad()\n            l = loss(net(X), y)\n            l.backward()\n            optimizer.step()\n        train_ls.append(float(loss(net(train_features), train_labels).to(\"cpu\").detach().numpy()))\n        gc.collect()\n        train_score.append(correlation_score(net(train_features).to(\"cpu\").detach().numpy(), train_labels.to(\"cpu\").detach().numpy()))\n        gc.collect()\n        if test_labels is not None:\n            test_ls.append(float(loss(net(test_features), test_labels).to(\"cpu\").detach().numpy()))\n            gc.collect()\n            test_score.append(correlation_score(net(test_features).to(\"cpu\").detach().numpy(), test_labels.to(\"cpu\").detach().numpy()))\n            gc.collect()\n        print(f\"train mse = {train_ls[-1]}, train corr =  {train_score[-1]}\")\n        print(f\"test mse = {test_ls[-1]}, test corr =  {test_score[-1]}\")\n    return train_ls, test_ls, train_score, test_score","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:13:40.865058Z","iopub.execute_input":"2022-08-27T05:13:40.865459Z","iopub.status.idle":"2022-08-27T05:13:40.877792Z","shell.execute_reply.started":"2022-08-27T05:13:40.865419Z","shell.execute_reply":"2022-08-27T05:13:40.876501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_nn_model():\n    net = nn.Sequential(nn.Linear(24, 512),\n                        nn.ReLU(),\n                        nn.Linear(512, 256),\n                        nn.ReLU(),\n                        nn.Linear(256, 256),\n                        nn.ReLU(),\n                        nn.Linear(256, 23418),\n                        )\n    def init_weights(m):\n        if type(m) == nn.Linear:\n            nn.init.normal_(m.weight, std=0.01)\n    net.apply(init_weights)\n\n    return net\n\nloss = nn.MSELoss()\nnet = get_nn_model()\ndevice=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nnet = net.to(device)\nloss = loss.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:13:42.795034Z","iopub.execute_input":"2022-08-27T05:13:42.795839Z","iopub.status.idle":"2022-08-27T05:13:42.90631Z","shell.execute_reply.started":"2022-08-27T05:13:42.795797Z","shell.execute_reply":"2022-08-27T05:13:42.90531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(multi_train_x)\ndf.std()","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:24:54.841954Z","iopub.execute_input":"2022-08-27T05:24:54.842625Z","iopub.status.idle":"2022-08-27T05:24:54.855645Z","shell.execute_reply.started":"2022-08-27T05:24:54.842585Z","shell.execute_reply":"2022-08-27T05:24:54.854249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nmulti_train_x = scaler.fit_transform(multi_train_x)","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:24:39.766831Z","iopub.execute_input":"2022-08-27T05:24:39.767468Z","iopub.status.idle":"2022-08-27T05:24:39.782684Z","shell.execute_reply.started":"2022-08-27T05:24:39.7674Z","shell.execute_reply":"2022-08-27T05:24:39.780137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"%%time\n# Cross-validation\n\nkf = KFold(n_splits=5, shuffle=True, random_state=1)\nscore_list = []\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(multi_train_x)):\n    model = None\n    gc.collect()\n    X_tr = multi_train_x[idx_tr] # creates a copy, https://numpy.org/doc/stable/user/basics.copies.html\n    y_tr = multi_train_y[idx_tr]\n    del idx_tr\n    \n    X_va = multi_train_x[idx_va]\n    y_va = multi_train_y[idx_va]\n    del idx_va\n    gc.collect()\n    \n    X_tr, y_tr, X_va, y_va = (torch.from_numpy(i).to(device) for i in [X_tr, y_tr, X_va, y_va])\n    \n    train_ls, test_ls, train_score, test_score = train(net, X_tr, y_tr, X_va, y_va, 10, 0.00001, 0, 128)\n\n    del X_tr, y_tr, X_va, y_va\n    gc.collect()\n\n    print(f\"Fold {fold}: train mse = {train_ls}, test corr =  {train_score}\")\n    print(f\"Fold {fold}: test mse = {test_ls}, test corr =  {test_score}\")\n    print(\"-------------------------\")\n    score_list.append((test_ls, test_score))\n\n# Show overall score\nresult_df = pd.DataFrame(score_list, columns=['mse', 'corrscore'])\nprint(f\"{Fore.GREEN}{Style.BRIGHT}{multi_train_x.shape} Average  mse = {result_df.mse.mean():.5f}; corr = {result_df.corrscore.mean():.3f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:29:09.325865Z","iopub.execute_input":"2022-08-27T05:29:09.326285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Cross-validation\n\nkf = KFold(n_splits=5, shuffle=True, random_state=1)\nscore_list = []\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(multi_train_x)):\n    model = None\n    gc.collect()\n    print(type(multi_train_x))\n    X_tr = multi_train_x[idx_tr] # creates a copy, https://numpy.org/doc/stable/user/basics.copies.html\n    y_tr = multi_train_y[idx_tr]\n    \n    print(X_tr.shape, y_tr.shape)\n    del idx_tr\n\n    model = Ridge(copy_X=False)\n    model.fit(X_tr, y_tr)\n    del X_tr, y_tr\n    gc.collect()\n\n    # We validate the model\n    X_va = multi_train_x[idx_va]\n    y_va = multi_train_y[idx_va]\n    del idx_va\n    y_va_pred = model.predict(X_va)\n    mse = mean_squared_error(y_va, y_va_pred)\n    corrscore = correlation_score(y_va, y_va_pred)\n    del X_va, y_va\n\n    print(f\"Fold {fold}: mse = {mse:.5f}, corr =  {corrscore:.3f}\")\n    score_list.append((mse, corrscore))\n\n# Show overall score\nresult_df = pd.DataFrame(score_list, columns=['mse', 'corrscore'])\nprint(f\"{Fore.GREEN}{Style.BRIGHT}{multi_train_x.shape} Average  mse = {result_df.mse.mean():.5f}; corr = {result_df.corrscore.mean():.3f}{Style.RESET_ALL}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-27T05:26:39.753582Z","iopub.execute_input":"2022-08-27T05:26:39.754372Z","iopub.status.idle":"2022-08-27T05:26:51.182786Z","shell.execute_reply.started":"2022-08-27T05:26:39.75433Z","shell.execute_reply":"2022-08-27T05:26:51.181323Z"},"trusted":true},"execution_count":null,"outputs":[]}]}