{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Here is the 5th solution (Multiome part). \n**Our highest private score is obtained by integrating 2 MLP models and 2 1D-CNN models. The method is very simple. If you read it, you will not regret it.**\n\nThis notebook includes a total of four models.\n\nThe inspiration of 1D-CNN comes from tmp. See the following discussion for details.\n\n[https://www.kaggle.com/competitions/lish-moa/discussion/202256)](http://)\n\nThanks for his sharing!","metadata":{}},{"cell_type":"code","source":"import sklearn\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom tqdm import tqdm\nimport copy\nimport os\n\nimport numpy as np\n\nfrom sklearn.decomposition import TruncatedSVD\n\nfrom scipy.sparse import csc_matrix\nimport logging\nfrom torch.utils.data.dataset import Dataset\nimport numpy as np\n\n\nDATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:08.700077Z","iopub.execute_input":"2022-11-18T01:50:08.700924Z","iopub.status.idle":"2022-11-18T01:50:11.214884Z","shell.execute_reply.started":"2022-11-18T01:50:08.700826Z","shell.execute_reply":"2022-11-18T01:50:11.213885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\nimport pandas as pd\n\nimport scipy\nimport scipy.sparse\n\nif not os.path.exists('/opt/conda/lib/python3.7/site-packages/tables'):\n    !pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.216871Z","iopub.execute_input":"2022-11-18T01:50:11.217361Z","iopub.status.idle":"2022-11-18T01:50:11.226803Z","shell.execute_reply.started":"2022-11-18T01:50:11.217332Z","shell.execute_reply":"2022-11-18T01:50:11.225804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df = pd.read_csv(FP_CELL_METADATA, index_col='cell_id')\nmetadata_df = metadata_df[metadata_df.technology==\"multiome\"]\nmetadata_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.228452Z","iopub.execute_input":"2022-11-18T01:50:11.228947Z","iopub.status.idle":"2022-11-18T01:50:11.658077Z","shell.execute_reply.started":"2022-11-18T01:50:11.228909Z","shell.execute_reply":"2022-11-18T01:50:11.657170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conditions = [\n    metadata_df['donor'].eq(27678) & metadata_df['day'].eq(2),\n    metadata_df['donor'].eq(27678) & metadata_df['day'].eq(3),\n    metadata_df['donor'].eq(27678) & metadata_df['day'].eq(4),\n    metadata_df['donor'].eq(27678) & metadata_df['day'].eq(7),\n    metadata_df['donor'].eq(27678) & metadata_df['day'].eq(10),\n    metadata_df['donor'].eq(13176) & metadata_df['day'].eq(2),\n    metadata_df['donor'].eq(13176) & metadata_df['day'].eq(3),\n    metadata_df['donor'].eq(13176) & metadata_df['day'].eq(4),\n    metadata_df['donor'].eq(13176) & metadata_df['day'].eq(7),\n    metadata_df['donor'].eq(13176) & metadata_df['day'].eq(10),\n    metadata_df['donor'].eq(31800) & metadata_df['day'].eq(2),\n    metadata_df['donor'].eq(31800) & metadata_df['day'].eq(3),\n    metadata_df['donor'].eq(31800) & metadata_df['day'].eq(4),\n    metadata_df['donor'].eq(31800) & metadata_df['day'].eq(7),\n    metadata_df['donor'].eq(31800) & metadata_df['day'].eq(10),\n    metadata_df['donor'].eq(32606) & metadata_df['day'].eq(2),\n    metadata_df['donor'].eq(32606) & metadata_df['day'].eq(3),\n    metadata_df['donor'].eq(32606) & metadata_df['day'].eq(4),\n    metadata_df['donor'].eq(32606) & metadata_df['day'].eq(7),\n    metadata_df['donor'].eq(32606) & metadata_df['day'].eq(10),\n    ]\n# create a list of the values we want to assign for each condition\nvalues = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]\n\n# create a new column and use np.select to assign values to it using our lists as arguments\nmetadata_df['comb'] = np.select(conditions, values)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.660941Z","iopub.execute_input":"2022-11-18T01:50:11.661359Z","iopub.status.idle":"2022-11-18T01:50:11.690719Z","shell.execute_reply.started":"2022-11-18T01:50:11.661322Z","shell.execute_reply":"2022-11-18T01:50:11.689844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.load('../input/multimodal-single-cell-as-sparse-matrix/train_multi_inputs_idxcol.npz', allow_pickle=True)\ncell_index = X['index']\nmeta = metadata_df.reindex(cell_index)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.692221Z","iopub.execute_input":"2022-11-18T01:50:11.692606Z","iopub.status.idle":"2022-11-18T01:50:11.828565Z","shell.execute_reply.started":"2022-11-18T01:50:11.692570Z","shell.execute_reply":"2022-11-18T01:50:11.827530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def NegativeCorrLoss(preds, targets):\n    \"\"\"Compute the correlation between each rows of the y_true and y_pred tensors.\n    Compatible with backpropagation.\n    \"\"\"\n    my = torch.mean(preds, dim=1)\n    my = torch.tile(torch.unsqueeze(my, dim=1), (1, targets.shape[1]))\n\n    ym = preds - my\n    r_num = torch.sum(torch.multiply(targets, ym), dim=1)\n    r_den = torch.sqrt(\n        torch.sum(torch.square(ym), dim=1) * float(targets.shape[-1])\n    )\n    r = torch.mean(r_num / r_den)\n    return -r\ndef correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.830252Z","iopub.execute_input":"2022-11-18T01:50:11.830647Z","iopub.status.idle":"2022-11-18T01:50:11.840400Z","shell.execute_reply.started":"2022-11-18T01:50:11.830610Z","shell.execute_reply":"2022-11-18T01:50:11.839259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, split, X_train, X_val, X_test, y_train, y_val):\n        self.split = split\n\n        if self.split == \"train\":\n            self.data = X_train\n            self.gt = y_train\n        elif self.split == \"val\":\n            self.data = X_val\n            self.gt = y_val\n        else:\n            self.data = X_test\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        if self.split == \"train\":\n            return self.data[idx], self.gt[idx]\n        elif self.split == \"val\":\n            return self.data[idx], 0\n        else:\n            return self.data[idx]","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.842334Z","iopub.execute_input":"2022-11-18T01:50:11.842694Z","iopub.status.idle":"2022-11-18T01:50:11.854031Z","shell.execute_reply.started":"2022-11-18T01:50:11.842659Z","shell.execute_reply":"2022-11-18T01:50:11.852941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, optimizer, dataloaders_dict,  true_test_mod2, scheduler, num_epochs):\n    best_mse = 100\n    best_model = 0\n    best_cor = 0\n    for epoch in range(num_epochs):\n        y_pred = []\n\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss, running_corrects = 0.0, 0\n\n            for inputs, gts in tqdm(dataloaders_dict[phase]):\n                inputs = inputs.cuda()\n                gts = gts.cuda()\n\n                optimizer.zero_grad()\n\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n\n                    if phase == 'train':\n                        loss = NegativeCorrLoss(outputs, gts)\n                        running_loss += loss.item() * inputs.size(0)\n                        loss.backward()\n                        optimizer.step()\n                    else:\n                        y_pred.extend(outputs.cpu().numpy())\n\n\n            if phase == \"train\":\n                epoch_loss = running_loss / len(dataloaders_dict[phase].dataset)\n                print('{} Loss: {:.4f}'.format(phase, epoch_loss))\n            else:\n                y_pred = np.array(y_pred)\n                cor = correlation_score(true_test_mod2, y_pred)\n                print('cor: ', cor)\n                if cor > best_cor:\n                    best_model = copy.deepcopy(model)\n                    best_cor = cor\n        scheduler.step(cor)\n    print(\"Best cor: \", best_cor)\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.856499Z","iopub.execute_input":"2022-11-18T01:50:11.857206Z","iopub.status.idle":"2022-11-18T01:50:11.869305Z","shell.execute_reply.started":"2022-11-18T01:50:11.857167Z","shell.execute_reply":"2022-11-18T01:50:11.868170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer(model, dataloader):\n    y_pred = []\n    model.eval()\n\n    for inputs in tqdm(dataloader):\n        inputs = inputs.cuda()\n\n        with torch.set_grad_enabled(False):\n            outputs = model(inputs)\n\n            y_pred.extend(outputs.cpu().numpy())\n\n    y_pred = np.array(y_pred)\n\n    \n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.870922Z","iopub.execute_input":"2022-11-18T01:50:11.871787Z","iopub.status.idle":"2022-11-18T01:50:11.883071Z","shell.execute_reply.started":"2022-11-18T01:50:11.871750Z","shell.execute_reply":"2022-11-18T01:50:11.881968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net1(nn.Module): \n    def __init__(self, dim_mod1, dim_mod2):\n        super(Net1, self).__init__()\n        self.input_ = nn.Linear(dim_mod1, 2048)\n        self.fc = nn.Linear(2048, 2048)\n        self.fc1 = nn.Linear(2048, 512)\n        self.dropout1 = nn.Dropout(p=0.25)\n        self.dropout2 = nn.Dropout(p=0.2)\n        self.dropout3 = nn.Dropout(p=0.25)\n        self.output = nn.Linear(512, dim_mod2)\n    def forward(self, x):\n        x = F.gelu(self.input_(x))\n        x = self.dropout1(x)\n        x = F.gelu(self.fc(x))\n        x = self.dropout2(x)\n        x = F.gelu(self.fc1(x))\n        x = self.dropout3(x)\n        x = F.gelu(self.output(x))\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.888542Z","iopub.execute_input":"2022-11-18T01:50:11.888834Z","iopub.status.idle":"2022-11-18T01:50:11.899283Z","shell.execute_reply.started":"2022-11-18T01:50:11.888810Z","shell.execute_reply":"2022-11-18T01:50:11.898366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net2(nn.Module): \n    def __init__(self, dim_mod1, dim_mod2):\n        super(Net2, self).__init__()\n        self.swish = Swish()\n        self.input_ = nn.Linear(dim_mod1, 2048)\n        self.fc = nn.Linear(2048, 2048)\n        self.fc1 = nn.Linear(2048, 512)\n        self.dropout1 = nn.Dropout(p=0.25)\n        self.dropout2 = nn.Dropout(p=0.2)\n        self.dropout3 = nn.Dropout(p=0.25)\n        self.output = nn.Linear(512, dim_mod2)\n    def forward(self, x):\n        x = self.swish(self.input_(x))\n        x = self.dropout1(x)\n        x = self.swish(self.fc(x))\n        x = self.dropout2(x)\n        x = self.swish(self.fc1(x))\n        x = self.dropout3(x)\n        x = self.swish(self.output(x))\n        return x\n    \nclass Swish(nn.Module):\n    def __init__(self,inplace=True):\n        super(Swish,self).__init__()\n        self.inplace=inplace\n    def forward(self,x):\n        if self.inplace:\n            x.mul_(torch.sigmoid(x))\n            return x\n        else:\n            return x*torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.900812Z","iopub.execute_input":"2022-11-18T01:50:11.901401Z","iopub.status.idle":"2022-11-18T01:50:11.913525Z","shell.execute_reply.started":"2022-11-18T01:50:11.901359Z","shell.execute_reply":"2022-11-18T01:50:11.912566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net3(nn.Module):\n        def __init__(self, num_features, num_targets, hidden_size):\n            super(Net3, self).__init__()\n            cha_1 = 256\n            cha_2 = 512\n            cha_3 = 512\n            \n            cha_1_reshape = int(hidden_size/cha_1)\n            cha_po_1 = int(hidden_size/cha_1/2)\n            cha_po_2 = int(hidden_size/cha_1/2/2) * cha_3\n            self.silu = nn.SiLU()\n            self.cha_1 = cha_1\n            self.cha_2 = cha_2\n            self.cha_3 = cha_3\n            self.cha_1_reshape = cha_1_reshape\n            self.cha_po_1 = cha_po_1\n            self.cha_po_2 = cha_po_2\n\n            self.dropout1 = nn.Dropout(0.1)\n            self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))\n\n            self.dropout_c1 = nn.Dropout(0.1)\n            self.conv1 = nn.utils.weight_norm(nn.Conv1d(cha_1,cha_2, kernel_size = 5, stride = 1, padding=2,  bias=False),dim=None)\n\n            self.ave_po_c1 = nn.AdaptiveAvgPool1d(output_size = cha_po_1)\n\n            self.dropout_c2 = nn.Dropout(0.1)\n            self.conv2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.dropout_c2_1 = nn.Dropout(0.3)\n            self.conv2_1 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.dropout_c2_2 = nn.Dropout(0.2)\n            self.conv2_2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_3, kernel_size = 5, stride = 1, padding=2, bias=True),dim=None)\n\n            self.max_po_c2 = nn.MaxPool1d(kernel_size=4, stride=2, padding=1)\n\n            self.flt = nn.Flatten()\n\n            self.dropout3 = nn.Dropout(0.2)\n            self.dense3 = nn.utils.weight_norm(nn.Linear(cha_po_2, num_targets))\n\n        def forward(self, x):\n\n            x = self.silu(self.dense1(x))\n\n            x = x.reshape(x.shape[0],self.cha_1,\n                          self.cha_1_reshape)\n\n            x = self.dropout_c1(x)\n            x = self.silu(self.conv1(x))\n\n            x = self.ave_po_c1(x)\n\n            x = self.dropout_c2(x)\n            x = self.silu(self.conv2(x))\n            x_s = x\n\n            x = self.dropout_c2_1(x)\n            x = self.silu(self.conv2_1(x))\n\n            x = self.dropout_c2_2(x)\n            x = self.silu(self.conv2_2(x))\n            x =  x * x_s\n\n            x = self.max_po_c2(x)\n\n            x = self.flt(x)\n\n            x = self.dropout3(x)\n            x = self.dense3(x)\n\n            return x\n","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.915747Z","iopub.execute_input":"2022-11-18T01:50:11.916539Z","iopub.status.idle":"2022-11-18T01:50:11.932795Z","shell.execute_reply.started":"2022-11-18T01:50:11.916503Z","shell.execute_reply":"2022-11-18T01:50:11.931673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net4(nn.Module):\n        def __init__(self, num_features, num_targets, hidden_size):\n            super(Net3, self).__init__()\n            cha_1 = 256\n            cha_2 = 512\n            cha_3 = 512\n            \n            cha_1_reshape = int(hidden_size/cha_1)\n            cha_po_1 = int(hidden_size/cha_1/2)\n            cha_po_2 = int(hidden_size/cha_1/2/2) * cha_3\n            self.cha_1 = cha_1\n            self.cha_2 = cha_2\n            self.cha_3 = cha_3\n            self.cha_1_reshape = cha_1_reshape\n            self.cha_po_1 = cha_po_1\n            self.cha_po_2 = cha_po_2\n\n            self.dropout1 = nn.Dropout(0.1)\n            self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))\n\n            self.dropout_c1 = nn.Dropout(0.1)\n            self.conv1 = nn.utils.weight_norm(nn.Conv1d(cha_1,cha_2, kernel_size = 5, stride = 1, padding=2,  bias=False),dim=None)\n\n            self.ave_po_c1 = nn.AdaptiveAvgPool1d(output_size = cha_po_1)\n\n            self.dropout_c2 = nn.Dropout(0.1)\n            self.conv2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.dropout_c2_1 = nn.Dropout(0.3)\n            self.conv2_1 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.dropout_c2_2 = nn.Dropout(0.2)\n            self.conv2_2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_3, kernel_size = 5, stride = 1, padding=2, bias=True),dim=None)\n\n            self.max_po_c2 = nn.MaxPool1d(kernel_size=4, stride=2, padding=1)\n\n            self.flt = nn.Flatten()\n\n            self.dropout3 = nn.Dropout(0.2)\n            self.dense3 = nn.utils.weight_norm(nn.Linear(cha_po_2, num_targets))\n\n        def forward(self, x):\n\n            x = F.hardswish(self.dense1(x))\n\n            x = x.reshape(x.shape[0],self.cha_1,\n                          self.cha_1_reshape)\n\n            x = self.dropout_c1(x)\n            x = F.hardswish(self.conv1(x))\n\n            x = self.ave_po_c1(x)\n\n            x = self.dropout_c2(x)\n            x = F.hardswish(self.conv2(x))\n            x_s = x\n\n            x = self.dropout_c2_1(x)\n            x = F.hardswish(self.conv2_1(x))\n\n            x = self.dropout_c2_2(x)\n            x = F.hardswish(self.conv2_2(x))\n            x =  x * x_s\n\n            x = self.max_po_c2(x)\n\n            x = self.flt(x)\n\n            x = self.dropout3(x)\n            x = self.dense3(x)\n\n            return x\n","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.934263Z","iopub.execute_input":"2022-11-18T01:50:11.934727Z","iopub.status.idle":"2022-11-18T01:50:11.952939Z","shell.execute_reply.started":"2022-11-18T01:50:11.934690Z","shell.execute_reply":"2022-11-18T01:50:11.951949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_ = 1\npreprocessing = 1","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.954332Z","iopub.execute_input":"2022-11-18T01:50:11.954768Z","iopub.status.idle":"2022-11-18T01:50:11.965939Z","shell.execute_reply.started":"2022-11-18T01:50:11.954730Z","shell.execute_reply":"2022-11-18T01:50:11.965020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if preprocessing == 1:\n    X = pd.read_csv('../input/multi-preprocessing/X_164_l2.csv').values\n    Xt = pd.read_csv('../input/multi-preprocessing/Xt_164_l2.csv').values\nelse:\n    X = pd.read_csv('../input/multi-preprocessing/X_164_max.csv').values\n    Xt = pd.read_csv('../input/multi-preprocessing/Xt_164_max.csv').values","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:11.967589Z","iopub.execute_input":"2022-11-18T01:50:11.967990Z","iopub.status.idle":"2022-11-18T01:50:21.216122Z","shell.execute_reply.started":"2022-11-18T01:50:11.967956Z","shell.execute_reply":"2022-11-18T01:50:21.215031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = pd.read_hdf(FP_MULTIOME_TRAIN_TARGETS).values\nY -= Y.mean(axis=1).reshape(-1, 1)\nY /= Y.std(axis=1).reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.217507Z","iopub.execute_input":"2022-11-18T01:50:21.217863Z","iopub.status.idle":"2022-11-18T01:50:21.358041Z","shell.execute_reply.started":"2022-11-18T01:50:21.217827Z","shell.execute_reply":"2022-11-18T01:50:21.356515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.astype('float32')\nXt = Xt.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.359250Z","iopub.status.idle":"2022-11-18T01:50:21.359851Z","shell.execute_reply.started":"2022-11-18T01:50:21.359588Z","shell.execute_reply":"2022-11-18T01:50:21.359612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport torch.nn.functional as F\nfrom sklearn.model_selection import GroupKFold\ny_pred = 0\nN_SPLITS_ANN = len(meta['comb'].value_counts())\nkf = GroupKFold(n_splits=N_SPLITS_ANN)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(X, groups=meta.comb)):\n        model = None\n        gc.collect()\n        X_train = X[idx_tr] \n        y_train = Y[idx_tr]\n        X_val = X[idx_va]\n        y_val = Y[idx_va]   \n        X_test = Xt\n        \n        if model_ == 1:\n            model = Net1(164,23418).cuda()\n        elif model_ == 2:\n            model = Net2(164,23418).cuda()\n        elif model_ == 3:\n            model = Net3(164,23418,1024).cuda()\n        else:\n            model = Net4(164,23418,1024).cuda()\n        lr = 1e-4\n        \n        optimizer_ft = optim.AdamW(model.parameters(), lr=lr)\n        \n        data = {x: CustomDataset(x, X_train, X_val, X_test, y_train, y_val) for x in ['train', 'val', 'test']}\n        dataloaders_dict = {\"train\": torch.utils.data.DataLoader(data[\"train\"], batch_size=512, shuffle=True, num_workers=8),\n                        \"val\": torch.utils.data.DataLoader(data[\"val\"], batch_size=512, shuffle=False, num_workers=8),\n                        \"test\": torch.utils.data.DataLoader(data[\"test\"], batch_size=512, shuffle=False, num_workers=8)}\n\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_ft, mode='max', patience=30, factor=0.1, verbose=True)\n        best_model_net = train_model(model, optimizer_ft, dataloaders_dict, y_val, scheduler, num_epochs=100)\n        y_pred = y_pred + infer(best_model_net, dataloaders_dict[\"test\"])","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.363950Z","iopub.status.idle":"2022-11-18T01:50:21.364739Z","shell.execute_reply.started":"2022-11-18T01:50:21.364486Z","shell.execute_reply":"2022-11-18T01:50:21.364511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Read the table of rows and columns required for submission\neval_ids = pd.read_parquet(\"../input/multimodal-single-cell-as-sparse-matrix/evaluation.parquet\")\n# Convert the string columns to more efficient categorical types\n#eval_ids.cell_id = eval_ids.cell_id.apply(lambda s: int(s, base=16))\neval_ids.cell_id = eval_ids.cell_id.astype(pd.CategoricalDtype())\neval_ids.gene_id = eval_ids.gene_id.astype(pd.CategoricalDtype())","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.366208Z","iopub.status.idle":"2022-11-18T01:50:21.366985Z","shell.execute_reply.started":"2022-11-18T01:50:21.366710Z","shell.execute_reply":"2022-11-18T01:50:21.366735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare an empty series which will be filled with predictions\nsubmission = pd.Series(name='target',\n                       index=pd.MultiIndex.from_frame(eval_ids), \n                       dtype=np.float32)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.368456Z","iopub.status.idle":"2022-11-18T01:50:21.369284Z","shell.execute_reply.started":"2022-11-18T01:50:21.368983Z","shell.execute_reply":"2022-11-18T01:50:21.369010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ny_columns = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_inputs_idxcol.npz\",\n                   allow_pickle=True)[\"columns\"]\n\ntest_index = np.load(\"../input/multimodal-single-cell-as-sparse-matrix/train_multi_inputs_idxcol.npz\",\n                    allow_pickle=True)[\"index\"]","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.370691Z","iopub.status.idle":"2022-11-18T01:50:21.371459Z","shell.execute_reply.started":"2022-11-18T01:50:21.371209Z","shell.execute_reply":"2022-11-18T01:50:21.371234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_dict = dict((k,v) for v,k in enumerate(test_index)) \nassert len(cell_dict)  == len(test_index)\n\ngene_dict = dict((k,v) for v,k in enumerate(y_columns))\nassert len(gene_dict) == len(y_columns)\neval_ids_cell_num = eval_ids.cell_id.apply(lambda x:cell_dict.get(x, -1))\neval_ids_gene_num = eval_ids.gene_id.apply(lambda x:gene_dict.get(x, -1))\n\nvalid_multi_rows = (eval_ids_gene_num !=-1) & (eval_ids_cell_num!=-1)\nsubmission.iloc[valid_multi_rows] = y_pred[eval_ids_cell_num[valid_multi_rows].to_numpy(),\neval_ids_gene_num[valid_multi_rows].to_numpy()]\ndel eval_ids_cell_num, eval_ids_gene_num, valid_multi_rows, eval_ids, test_index, y_columns\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.372820Z","iopub.status.idle":"2022-11-18T01:50:21.373567Z","shell.execute_reply.started":"2022-11-18T01:50:21.373305Z","shell.execute_reply":"2022-11-18T01:50:21.373330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.reset_index(drop=True, inplace=True)\nsubmission.index.name = 'row_id'","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.374944Z","iopub.status.idle":"2022-11-18T01:50:21.375707Z","shell.execute_reply.started":"2022-11-18T01:50:21.375443Z","shell.execute_reply":"2022-11-18T01:50:21.375467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T01:50:21.377109Z","iopub.status.idle":"2022-11-18T01:50:21.377892Z","shell.execute_reply.started":"2022-11-18T01:50:21.377620Z","shell.execute_reply":"2022-11-18T01:50:21.377646Z"},"trusted":true},"execution_count":null,"outputs":[]}]}