{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"datasetVersion","sourceId":9567507,"datasetId":5831279,"databundleVersionId":9786273},{"sourceType":"modelInstanceVersion","sourceId":129189,"databundleVersionId":9786684,"modelInstanceId":108851}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport math\nimport copy\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torchvision\n\n# import skimage\nfrom skimage.io import imread\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score","metadata":{"_uuid":"65655386f03bf88bb08a5086a45aac0e2a528b3d","execution":{"iopub.status.busy":"2024-10-09T14:12:21.209061Z","iopub.execute_input":"2024-10-09T14:12:21.210058Z","iopub.status.idle":"2024-10-09T14:12:27.364387Z","shell.execute_reply.started":"2024-10-09T14:12:21.210002Z","shell.execute_reply":"2024-10-09T14:12:27.363546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom nilearn.connectome import ConnectivityMeasure\n\n\ndef get_connectome(timeseries: np.ndarray,\n                   conn_type: str = 'corr') -> np.ndarray:\n    if conn_type == 'corr':\n        conn = ConnectivityMeasure(kind='correlation', standardize=False).fit_transform(timeseries)\n        conn[conn == 1] = 0.999999\n\n        for i in conn:\n            np.fill_diagonal(i, 0)\n\n        conn = np.arctanh(conn)\n\n    else:\n        raise NotImplementedError\n\n    return conn","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:27.365966Z","iopub.execute_input":"2024-10-09T14:12:27.366419Z","iopub.status.idle":"2024-10-09T14:12:27.904985Z","shell.execute_reply.started":"2024-10-09T14:12:27.366386Z","shell.execute_reply":"2024-10-09T14:12:27.903999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configurations**","metadata":{"_uuid":"ea585d58a85d22857cc8dc5911a23fd3be78ffb3"}},{"cell_type":"code","source":"bnu_series_path = '/kaggle/input/aidao2/data/ts_cut/HCPex/bnu{}.npy'\nbnu_labels_path = '/kaggle/input/aidao2/data/ts_cut/HCPex/bnu.csv'\nihb_series_path = '/kaggle/input/aidao2/data/ts_cut/HCPex/ihb.npy'\nihb_labels_path = '/kaggle/input/aidao2/data/ts_cut/HCPex/ihb.csv'\n\nX_bnu = np.concatenate([np.load(bnu_series_path.format(i)) for i in (1, 2)], axis=0)\nprint(X_bnu.shape)\nY_bnu = pd.read_csv(bnu_labels_path)\nprint(Y_bnu.shape)\nX_ihb = np.load(ihb_series_path)\nprint(X_ihb.shape)\nY_ihb = pd.read_csv(ihb_labels_path)\nprint(Y_ihb.shape)\n\n# time series have different length\n# by the way ``get_connectome`` reduces them to matrices 419x419\n\n# X_bnu = get_connectome(X_bnu)\n# print(X_bnu.shape)\n# X_ihb = get_connectome(X_ihb)\n\n# concat the train data\n# X = np.concatenate([X_bnu, X_ihb])\n# Y = np.concatenate([Y_bnu, Y_ihb])\n\n# let's split data into train and validation\n\n# x_train, x_validate, y_train, y_validate = train_test_split(X, Y,\n#                                                             test_size=0.1, random_state=10)\nx1_train, x1_validate, y1_train, y1_validate = train_test_split(X_bnu, Y_bnu,\n                                                            test_size=0.1, random_state=10)\n\nx2_train, x2_validate, y2_train, y2_validate = train_test_split(X_ihb, Y_ihb,\n                                                            test_size=0.1, random_state=10)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:27.906142Z","iopub.execute_input":"2024-10-09T14:12:27.906618Z","iopub.status.idle":"2024-10-09T14:12:29.114148Z","shell.execute_reply.started":"2024-10-09T14:12:27.906584Z","shell.execute_reply":"2024-10-09T14:12:29.113144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open(\"/kaggle/working/x_train.npy\", \"wb\") as f:\n#     np.save(f, arr=x_train)\n\n# with open(\"/kaggle/working/x_validate.npy\", \"wb\") as f:\n#     np.save(f, arr=x_validate)\n\n# with open(\"/kaggle/working/y_train.npy\", \"wb\") as f:\n#     np.save(f, arr=y_train.reshape(-1))\n\n# with open(\"/kaggle/working/y_validate.npy\", \"wb\") as f:\n#     np.save(f, arr=y_validate.reshape(-1))","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:29.116247Z","iopub.execute_input":"2024-10-09T14:12:29.116583Z","iopub.status.idle":"2024-10-09T14:12:29.121186Z","shell.execute_reply.started":"2024-10-09T14:12:29.116547Z","shell.execute_reply":"2024-10-09T14:12:29.120140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 5\nUSE_GPU = True","metadata":{"_uuid":"875595f5b6e66679d8d765216ab75ed952176c67","execution":{"iopub.status.busy":"2024-10-09T14:12:29.122255Z","iopub.execute_input":"2024-10-09T14:12:29.122525Z","iopub.status.idle":"2024-10-09T14:12:29.131341Z","shell.execute_reply.started":"2024-10-09T14:12:29.122495Z","shell.execute_reply":"2024-10-09T14:12:29.130456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load labels**","metadata":{"_uuid":"39c14c5d34d1ac13c7a518dec23d2a61bdf609f7"}},{"cell_type":"code","source":"import numpy as np\nfrom scipy.stats import zscore\nfrom sklearn.decomposition import PCA\nimport math\n\nclass EyesDataset(torch.utils.data.Dataset):\n    \"\"\"\n    This is our custom dataset class which will load the images, perform transforms on them,\n    and load their corresponding labels.\n    \"\"\"\n    \n    def __init__(self, data, labels, train=True):\n#         self.pca = pca\n        self.X = np.concatenate([np.load(d) for d in data], axis=0)\n        if train:\n            self.Y = np.concatenate([np.load(d) for d in labels], axis=0)\n        else:\n            self.Y = None\n\n        # time series have different length\n        # by the way ``get_connectome`` reduces them to matrices 419x419\n        self.train = train\n#         self.X = get_connectome(self.X)\n        self.X = self._preprocess(self.X)\n        \n#         if self.train:\n#             self.pca = PCA(n_components=reduction)\n#             self.X = self.pca.fit_transform(self.X)\n#         else:\n#             self.X = self.pca.transform(self.X)\n        print(self.X.shape)\n        self.X = torch.from_numpy(self.X).float()\n#         self.X = self.X.reshape(..., int(math.sqrt(reduction)), int(math.sqrt(reduction)))\n        \n    def _preprocess(self, x):\n        vecs = zscore(x, axis=0)\n\n        for i in vecs:\n            np.fill_diagonal(i, 0)\n\n        return vecs\n    \n    def __getitem__(self, idx):\n#         a = self.X[idx].view(1, self.X.shape[1], self.X.shape[2])\n        sample = {\n            \"image\": self.X[idx],\n        }\n        if self.train:\n            sample[\"label\"] = self.Y[idx]\n        else:\n            sample[\"label\"] = None\n        \n        return sample\n    \n    def __len__(self):\n        return len(self.X)","metadata":{"_uuid":"d400510a381dd032a986bb552fa3e3d3bbab4672","execution":{"iopub.status.busy":"2024-10-09T14:13:53.267276Z","iopub.execute_input":"2024-10-09T14:13:53.267925Z","iopub.status.idle":"2024-10-09T14:13:53.278209Z","shell.execute_reply.started":"2024-10-09T14:13:53.267884Z","shell.execute_reply":"2024-10-09T14:13:53.277193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EyesDatasetPCA(torch.utils.data.Dataset):\n    \"\"\"\n    This is our custom dataset class which will load the images, perform transforms on them,\n    and load their corresponding labels.\n    \"\"\"\n    \n    def __init__(self, data, labels, pca=None, n_components=0.9, train=True):\n        self.pca = pca\n        self.X = np.concatenate([np.load(d) for d in data], axis=0)\n        if train:\n            self.Y = np.concatenate([np.load(d) for d in labels], axis=0)\n        else:\n            self.Y = None\n\n        # time series have different length\n        # by the way ``get_connectome`` reduces them to matrices 419x419\n        self.train = train\n#         self.X = get_connectome(self.X)\n        self.X = self._preprocess(self.X)\n        \n        if self.pca is None:\n            self.pca = PCA(n_components=n_components)\n            self.X = self.pca.fit_transform(self.X)\n        else:\n            self.X = self.pca.transform(self.X)\n        print(self.X.shape)\n        self.X = torch.from_numpy(self.X).float()\n#         self.X = self.X.reshape(..., int(math.sqrt(reduction)), int(math.sqrt(reduction)))\n        \n    def _preprocess(self, x):\n        vecs = zscore(x, axis=0)\n\n        for i in vecs:\n            np.fill_diagonal(i, 0)\n        \n        vecs = vecs.reshape((x.shape[0], -1))\n        return vecs\n    \n    def __getitem__(self, idx):\n        a = self.X[idx]\n        sample = {\n            \"image\": a,\n        }\n        if self.train:\n            sample[\"label\"] = self.Y[idx]\n        else:\n            sample[\"label\"] = None\n        \n        return sample\n    \n    def __len__(self):\n        return len(self.X)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:29.180149Z","iopub.execute_input":"2024-10-09T14:12:29.180939Z","iopub.status.idle":"2024-10-09T14:12:29.191511Z","shell.execute_reply.started":"2024-10-09T14:12:29.180904Z","shell.execute_reply":"2024-10-09T14:12:29.190439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EyesDatasetTransformer(torch.utils.data.Dataset):\n    \"\"\"\n    This is our custom dataset class which will load the images, perform transforms on them,\n    and load their corresponding labels.\n    \"\"\"\n    \n    def __init__(self, data1, data2, labels1, labels2):\n        self.X = [torch.from_numpy(d).float() for d in data1] + [torch.from_numpy(d).float() for d in data2]\n        self.Y = np.concatenate([labels1, labels2])\n    \n    def __getitem__(self, idx):\n        a = self.X[idx]\n        length = a.shape[0]\n        a = torch.nn.functional.pad(a, (0, 0, 0, 240 - a.shape[0]), mode='constant', value=-100)\n        \n        sample = {\n            \"image\": a,\n            \"length\": length,\n        }\n        sample[\"label\"] = self.Y[idx]\n        \n        return sample\n    \n    def __len__(self):\n        return len(self.X)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:23:06.592001Z","iopub.execute_input":"2024-10-09T14:23:06.592853Z","iopub.status.idle":"2024-10-09T14:23:06.600908Z","shell.execute_reply.started":"2024-10-09T14:23:06.592808Z","shell.execute_reply":"2024-10-09T14:23:06.599923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Image tranformation pipeline**","metadata":{"_uuid":"241df72febc76df14d21be298435bac5df1a546f"}},{"cell_type":"code","source":"# transform_pipe = torchvision.transforms.Compose([\n#     torchvision.transforms.ToPILImage(), # Convert np array to PILImage\n    \n#     # Resize image to 224 x 224 as required by most vision models\n#     torchvision.transforms.Resize(\n#         size=(224, 224)\n#     ),\n    \n#     # Convert PIL image to tensor with image values in [0, 1]    \n#     torchvision.transforms.Normalize(\n#         mean=[0.485, 0.456, 0.406],\n#         std=[0.229, 0.224, 0.225]\n#     )\n# ])","metadata":{"_uuid":"c51370550ec0a7a92fd8fc0ebedd0a02412cfa22","execution":{"iopub.status.busy":"2024-10-09T14:12:31.755736Z","iopub.execute_input":"2024-10-09T14:12:31.756630Z","iopub.status.idle":"2024-10-09T14:12:31.761923Z","shell.execute_reply.started":"2024-10-09T14:12:31.756591Z","shell.execute_reply":"2024-10-09T14:12:31.760860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = EyesDataset(\n#     data=['/kaggle/working/x_train.npy'], \n#     labels=['/kaggle/working/y_train.npy'], \n#     train=True, \n# #     pca=None, \n# #     reduction=224 * 224,\n# )","metadata":{"_uuid":"69c25803dccad729a4bfb95bdf032514e7b92064","execution":{"iopub.status.busy":"2024-10-09T14:14:09.486690Z","iopub.execute_input":"2024-10-09T14:14:09.487103Z","iopub.status.idle":"2024-10-09T14:14:10.073951Z","shell.execute_reply.started":"2024-10-09T14:14:09.487065Z","shell.execute_reply":"2024-10-09T14:14:10.072834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = EyesDatasetPCA(\n#     data=['/kaggle/working/x_train.npy'], \n#     labels=['/kaggle/working/y_train.npy'], \n#     train=True,\n#     pca=None,\n#     n_components=64\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:32.535844Z","iopub.execute_input":"2024-10-09T14:12:32.536200Z","iopub.status.idle":"2024-10-09T14:12:32.540430Z","shell.execute_reply.started":"2024-10-09T14:12:32.536166Z","shell.execute_reply":"2024-10-09T14:12:32.539491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = EyesDatasetTransformer(\n    data1=x1_train,\n    data2=x2_train,\n    labels1=y1_train,\n    labels2=y2_train\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:23:09.547940Z","iopub.execute_input":"2024-10-09T14:23:09.548686Z","iopub.status.idle":"2024-10-09T14:23:09.581538Z","shell.execute_reply.started":"2024-10-09T14:23:09.548644Z","shell.execute_reply":"2024-10-09T14:23:09.580668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_data = EyesDataset(\n#     data=['/kaggle/working/x_validate.npy'],\n#     labels=['/kaggle/working/y_validate.npy'],\n#     train=True,\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:14:15.785845Z","iopub.execute_input":"2024-10-09T14:14:15.786603Z","iopub.status.idle":"2024-10-09T14:14:15.855976Z","shell.execute_reply.started":"2024-10-09T14:14:15.786561Z","shell.execute_reply":"2024-10-09T14:14:15.854836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_data = EyesDatasetPCA(\n#     data=['/kaggle/working/x_validate.npy'],\n#     labels=['/kaggle/working/y_validate.npy'],\n#     train=True,\n#     pca=train_data.pca\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:33.840508Z","iopub.execute_input":"2024-10-09T14:12:33.840909Z","iopub.status.idle":"2024-10-09T14:12:33.845185Z","shell.execute_reply.started":"2024-10-09T14:12:33.840872Z","shell.execute_reply":"2024-10-09T14:12:33.844065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_data = EyesDatasetTransformer(\n    data1=x1_validate,\n    data2=x2_validate,\n    labels1=y1_validate,\n    labels2=y2_validate\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:23:12.357655Z","iopub.execute_input":"2024-10-09T14:23:12.358049Z","iopub.status.idle":"2024-10-09T14:23:12.367372Z","shell.execute_reply.started":"2024-10-09T14:23:12.358011Z","shell.execute_reply":"2024-10-09T14:23:12.366007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The training dataset loader will randomly sample from the train samples**","metadata":{"_uuid":"219bf456dfd642daea9cc82a77ff159071debe49"}},{"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(\n    train_data,\n    batch_size=16,\n#     sampler=torch.utils.data.SubsetRandomSampler(\n#         train_indices\n#     )\n    shuffle=True,\n#     num_workers=8\n)","metadata":{"_uuid":"0b26dc3d91fa365d8f23a8667a48647c2d27fbfc","execution":{"iopub.status.busy":"2024-10-09T14:23:13.131728Z","iopub.execute_input":"2024-10-09T14:23:13.132126Z","iopub.status.idle":"2024-10-09T14:23:13.137414Z","shell.execute_reply.started":"2024-10-09T14:23:13.132090Z","shell.execute_reply":"2024-10-09T14:23:13.136243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The training dataset loader will randomly sample from the test samples**","metadata":{"_uuid":"48375a7572ee065377f8fa3a18ade4553c17edcb"}},{"cell_type":"code","source":"val_loader = torch.utils.data.DataLoader(\n    val_data,\n    batch_size=17,\n#     sampler=torch.utils.data.SubsetRandomSampler(\n#         test_indices\n#     )\n    shuffle=False,\n#     num_workers=8\n)","metadata":{"_uuid":"24d1263c31ff663b5c2ff168d3b81ec0fa86f72b","execution":{"iopub.status.busy":"2024-10-09T14:23:14.228339Z","iopub.execute_input":"2024-10-09T14:23:14.229175Z","iopub.status.idle":"2024-10-09T14:23:14.233957Z","shell.execute_reply.started":"2024-10-09T14:23:14.229127Z","shell.execute_reply":"2024-10-09T14:23:14.232842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = {\n    \"train\": train_loader,\n    \"val\": val_loader\n}","metadata":{"_uuid":"94cf65bc49ab5b1881d9d571cf5f29ed287d2432","execution":{"iopub.status.busy":"2024-10-09T14:23:16.290050Z","iopub.execute_input":"2024-10-09T14:23:16.290446Z","iopub.status.idle":"2024-10-09T14:23:16.294773Z","shell.execute_reply.started":"2024-10-09T14:23:16.290412Z","shell.execute_reply":"2024-10-09T14:23:16.293773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"a5094af5c399ef22267b9145a540c7f6407bcbfd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model definition**","metadata":{"_uuid":"76628ba5dd13103c32f2c1ad3361185676b121d2"}},{"cell_type":"markdown","source":"**Replace the final fully connected layer to suite the problem**","metadata":{"_uuid":"a6d19b75eb04fe2d3f837fa8d8bac0c487cf1d52"}},{"cell_type":"code","source":"# model = torchvision.models.resnet50()\n# model.load_state_dict(torch.load(\"/kaggle/input/resnet50/pytorch/default/1/resnet50-0676ba61.pth\"))\n# model.fc = torch.nn.Sequential(\n#     torch.nn.Linear(\n#         in_features=256,\n#         out_features=1\n#     ),\n#     torch.nn.Sigmoid()\n# )","metadata":{"_uuid":"8f91bc25091c8c78c9fba5f282ff844d23dfa524","execution":{"iopub.status.busy":"2024-10-07T15:11:45.045850Z","iopub.execute_input":"2024-10-07T15:11:45.046230Z","iopub.status.idle":"2024-10-07T15:11:45.553963Z","shell.execute_reply.started":"2024-10-07T15:11:45.046191Z","shell.execute_reply":"2024-10-07T15:11:45.553018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = torch.nn.Sequential(\n#     torch.nn.Linear(\n#         in_features=111,\n#         out_features=256,\n#     ),\n#     torch.nn.ReLU(),\n#     torch.nn.Linear(\n#         in_features=256,\n#         out_features=256,\n#     ),\n#     torch.nn.ReLU(),\n#     torch.nn.Linear(\n#         in_features=256,\n#         out_features=1,\n#     ),\n#     torch.nn.Sigmoid()\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-07T16:36:58.195112Z","iopub.execute_input":"2024-10-07T16:36:58.195934Z","iopub.status.idle":"2024-10-07T16:36:58.203011Z","shell.execute_reply.started":"2024-10-07T16:36:58.195880Z","shell.execute_reply":"2024-10-07T16:36:58.202058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\ndef generate_square_mask(sz, device):\n    mask = (torch.triu(torch.ones((sz, sz), device=device)) == 1).transpose(0, 1)\n    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n    return mask\n\ndef create_mask(x, pad_idx):\n    tgt_padding_mask = (x[:, :, 0] == pad_idx)\n    return tgt_padding_mask","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:01:10.894379Z","iopub.execute_input":"2024-10-09T15:01:10.895155Z","iopub.status.idle":"2024-10-09T15:01:10.901631Z","shell.execute_reply.started":"2024-10-09T15:01:10.895114Z","shell.execute_reply":"2024-10-09T15:01:10.900473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyTransformer(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        encoder_layer = torch.nn.TransformerEncoderLayer(\n            d_model=64,\n            nhead=2,\n            dim_feedforward=128,\n            dropout=0.1,\n            batch_first=True,\n        )\n        self.transformer = torch.nn.TransformerEncoder(\n            encoder_layer,\n            5\n        )\n        \n        self.linear = torch.nn.Linear(in_features=419, out_features=64)\n        self.norm = torch.nn.LayerNorm(64)\n        self.clf = torch.nn.Linear(in_features=64, out_features=2)\n        self.sigmoid = torch.nn.Softmax()\n        \n    def forward(self, x):\n        mask = create_mask(x, -100)\n        out = self.linear(x)\n        out = self.norm(out)\n        out = self.transformer(src=out, src_key_padding_mask=mask)\n        out = out.mean(1)\n        return self.sigmoid(self.clf(out))","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:03:16.834376Z","iopub.execute_input":"2024-10-09T15:03:16.835086Z","iopub.status.idle":"2024-10-09T15:03:16.843840Z","shell.execute_reply.started":"2024-10-09T15:03:16.835042Z","shell.execute_reply":"2024-10-09T15:03:16.842861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MyTransformer()","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:03:16.989280Z","iopub.execute_input":"2024-10-09T15:03:16.989698Z","iopub.status.idle":"2024-10-09T15:03:17.005854Z","shell.execute_reply.started":"2024-10-09T15:03:16.989659Z","shell.execute_reply":"2024-10-09T15:03:17.004938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model training**","metadata":{"_uuid":"3f29b180dd8ad7e3a23ad495fadaa6c41587777d"}},{"cell_type":"code","source":"# model.layer2 = torch.nn.Identity()\n# model.layer3 = torch.nn.Identity()\n# model.layer4 = torch.nn.Identity()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:02:46.721578Z","iopub.execute_input":"2024-10-09T15:02:46.722290Z","iopub.status.idle":"2024-10-09T15:02:46.726179Z","shell.execute_reply.started":"2024-10-09T15:02:46.722249Z","shell.execute_reply":"2024-10-09T15:02:46.725199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for param in model.parameters():\n#     param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:02:46.951746Z","iopub.execute_input":"2024-10-09T15:02:46.952469Z","iopub.status.idle":"2024-10-09T15:02:46.956477Z","shell.execute_reply.started":"2024-10-09T15:02:46.952428Z","shell.execute_reply":"2024-10-09T15:02:46.955409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for param in model.fc.parameters():\n#     print(param)\n#     param.requires_grad = True","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:02:47.177295Z","iopub.execute_input":"2024-10-09T15:02:47.177700Z","iopub.status.idle":"2024-10-09T15:02:47.182297Z","shell.execute_reply.started":"2024-10-09T15:02:47.177662Z","shell.execute_reply":"2024-10-09T15:02:47.181255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 500\nUSE_GPU = True","metadata":{"execution":{"iopub.status.busy":"2024-10-09T15:03:20.783621Z","iopub.execute_input":"2024-10-09T15:03:20.784049Z","iopub.status.idle":"2024-10-09T15:03:20.788769Z","shell.execute_reply.started":"2024-10-09T15:03:20.784009Z","shell.execute_reply":"2024-10-09T15:03:20.787657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nif USE_GPU:\n    model = model.cuda() # Should be called before instantiating optimizer according to docs: https://pytorch.org/docs/stable/nn.html\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-8)\ncriterion = torch.nn.CrossEntropyLoss()\nscheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, 1e-1, total_steps=EPOCHS * len(train_loader), pct_start=0.1, anneal_strategy='cos')\n\nbest_model_wts = copy.deepcopy(model.state_dict())\nbest_acc = 0.0\n\nfor i in range(EPOCHS):\n    for phase in [\"train\", \"val\"]:\n        if phase == \"train\":\n            model.train()\n        else:\n            model.eval()\n        \n        samples = 0\n        loss_sum = 0\n        correct_sum = 0\n        for j, batch in enumerate(dataloaders[phase]):\n            X = batch[\"image\"]\n            labels = batch[\"label\"]\n            if USE_GPU:\n                X = X.cuda()\n                labels = labels.cuda()\n\n            optimizer.zero_grad()\n\n            with torch.set_grad_enabled(phase == 'train'):\n                y = model(X)\n                loss = criterion(\n                    y, \n                    labels.view(-1).long()\n                )\n\n                if phase == \"train\":\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    \n                loss_sum += loss.item() * X.shape[0] # We need to multiple by batch size as loss is the mean loss of the samples in the batch\n                samples += X.shape[0]\n                num_corrects = torch.sum(y.argmax(1) == labels.view(-1, 1).float())\n                correct_sum += num_corrects\n                \n                # Print batch statistics every 50 batches\n                if j % 50 == 49 and phase == \"train\":\n                    print(\"{}:{} - loss: {}, acc: {}\".format(\n                        i + 1, \n                        j + 1, \n                        float(loss_sum) / float(samples), \n                        float(correct_sum) / float(samples)\n                    ))\n                \n        # Print epoch statistics\n        epoch_acc = float(correct_sum) / float(samples)\n        epoch_loss = float(loss_sum) / float(samples)\n        print(\"epoch: {} - {} loss: {}, {} acc: {}\".format(i + 1, phase, epoch_loss, phase, epoch_acc))\n        \n        # Deep copy the model\n        if phase == \"val\" and epoch_acc > best_acc:\n            print(\"SAVING...\")\n            best_acc = epoch_acc\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(best_model_wts, f\"transformer_{str(best_acc)[:5]}.pth\")","metadata":{"_uuid":"d59db9eaf79310f397694fb80235d444c0e87ad2","execution":{"iopub.status.busy":"2024-10-09T15:03:20.919804Z","iopub.execute_input":"2024-10-09T15:03:20.920228Z","iopub.status.idle":"2024-10-09T15:03:33.609753Z","shell.execute_reply.started":"2024-10-09T15:03:20.920179Z","shell.execute_reply":"2024-10-09T15:03:33.608298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Persist latest model**","metadata":{"_uuid":"868d7222941e9f12ccb81d2236b5f6fc5343b67c"}},{"cell_type":"code","source":"import pickle\npkl_filename = \"/kaggle/working/pca.pkl\"\nwith open(pkl_filename, 'wb') as file:\n    pickle.dump(train_data.pca, file)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T16:21:56.673322Z","iopub.execute_input":"2024-10-07T16:21:56.674021Z","iopub.status.idle":"2024-10-07T16:21:57.226659Z","shell.execute_reply.started":"2024-10-07T16:21:56.673980Z","shell.execute_reply":"2024-10-07T16:21:57.225687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! /kaggle/working/kaggle/working/","metadata":{"_uuid":"b088d1389ec07e2062ad8ee021d090206adade37","execution":{"iopub.status.busy":"2024-10-07T16:14:08.020155Z","iopub.execute_input":"2024-10-07T16:14:08.021052Z","iopub.status.idle":"2024-10-07T16:14:09.053241Z","shell.execute_reply.started":"2024-10-07T16:14:08.021009Z","shell.execute_reply":"2024-10-07T16:14:09.052263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reconstruct model from saved weights**","metadata":{"_uuid":"a71c3ae63bb75481a42d72076a318f87f7c1f781"}},{"cell_type":"code","source":"model1 = torchvision.models.resnet50()\nmodel1.fc = torch.nn.Sequential(\n    torch.nn.Linear(\n        in_features=2048,\n        out_features=1\n    ),\n    torch.nn.Sigmoid()\n)\nmodel1.load_state_dict(torch.load(\"resnet50.pth\"))","metadata":{"_uuid":"f47a9938a9696fa2d51e1087efb2d0a5bcd7689c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make predictions**","metadata":{"_uuid":"eb5f948c07f5a13ea174f0c90d86d95f12efc7a6"}},{"cell_type":"code","source":"test_data = HistopathologicCancerDataset(\n    img_dir=\"../input/test/\",\n    transform=transform_pipe\n)","metadata":{"_uuid":"b27cb59293b4645d4421fae368195f59c792c5b0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader1 = torch.utils.data.DataLoader(\n    test_data,\n    batch_size=64,\n#     shuffle=True,\n#     num_workers=8\n)","metadata":{"_uuid":"69726b65aa54a73effdc571cafa1880b24c83603"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.eval()\nif USE_GPU:\n    model1 = model1.cuda()\n\nids_all = []\npredictions = []\n\nfor j, batch in enumerate(test_loader1):\n    X = batch[\"image\"]\n    ids = batch[\"id\"]\n    if USE_GPU:\n        X = X.cuda()\n    \n    for _id in ids:\n        ids_all.append(_id)\n\n    with torch.set_grad_enabled(False):\n        y_pred = model1(X)\n        predictions.append((y_pred >= 0.5).float().cpu().numpy())\n        \nprint(\"Done making predictions!\")","metadata":{"_uuid":"89bdaad8df07738b0af1c8d536f62ef7207165c6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions = pd.DataFrame({\n    \"id\": ids_all,\n    \"label\": np.concatenate(predictions).reshape(-1,).astype(\"int\")\n}).set_index(\"id\")","metadata":{"_uuid":"b4872db3ea679655a43d7c24aa79b0583a2016ba"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions.head()","metadata":{"_uuid":"4bfe438e6e6a8efb18fe12c59c56aed3bd4150b3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions.to_csv(\"submissions.csv\")","metadata":{"_uuid":"3c726f83f64226d321af3a15d8149960679aab20"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"c958435d7f7fc5298fc4d5c288d9ec871a8ba9e5"},"execution_count":null,"outputs":[]}]}