{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14077258,"sourceType":"datasetVersion","datasetId":8961306}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2 as cv\nimport matplotlib.pyplot as plt\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nDATA_PATH = \"/kaggle/input/physionet-ecg-image-digitization\"\ntrain_df = pd.read_csv(DATA_PATH  + '/train.csv')\ntest_df = pd.read_csv(DATA_PATH + '/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-10T06:27:25.493748Z","iopub.execute_input":"2025-12-10T06:27:25.494389Z","iopub.status.idle":"2025-12-10T06:27:31.898065Z","shell.execute_reply.started":"2025-12-10T06:27:25.494359Z","shell.execute_reply":"2025-12-10T06:27:31.897046Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train and Evaluate the following models, compare results\n\n##    The models are used to classify image patches that either contain a ECG waveform, or do not","metadata":{}},{"cell_type":"markdown","source":"## Data Prep","metadata":{}},{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport numpy as np\nimport glob\nimport gc\nimport pickle\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import accuracy_score\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n# -------------------------------\n# Config\n# -------------------------------\nparquet_files = [\"/kaggle/input/complete-patches-data-physionet-2025/patches_batch_67.parquet\"]\nbatch_size = 1024\nimage_col = \"image_patch\"  # each row is a flattened image array\nlabel_col = \"labelClf\"\ntrain_fraction = 0.8\nepochs = 5\n\n\ndef parquet_batch_generator(file_paths, image_col, label_col):\n    for f in file_paths:\n        pf = pq.ParquetFile(f)\n        for batch in pf.iter_batches(batch_size=batch_size):\n            df_batch = batch.to_pandas()\n            # Extract only necessary columns\n            X_batch = np.stack(df_batch[image_col].values)  # shape [batch, features]\n            y_batch = df_batch[label_col].to_numpy()\n            yield X_batch, y_batch\n\n\ndef split_batch(X, y, train_frac=0.8):\n    mask = np.random.rand(len(X)) < train_frac\n    return X[mask], y[mask], X[~mask], y[~mask]\n\n\nprint(\"Training Random Forest...\")\nrf_X_list, rf_y_list = [], []\n\nfor X_batch, y_batch in parquet_batch_generator(parquet_files, image_col, label_col):\n    X_train, y_train, _, _ = split_batch(X_batch, y_batch, train_fraction)\n    if len(X_train) == 0:\n        continue\n    rf_X_list.append(X_train)\n    rf_y_list.append(y_train)\n\nif len(rf_X_list) == 0:\n    raise ValueError(\"No training data for Random Forest!\")\n\nX_rf = np.vstack(rf_X_list)\ny_rf = np.concatenate(rf_y_list)\n\nrf = RandomForestClassifier(n_estimators=100)\nrf.fit(X_rf, y_rf)\n\n\nall_preds, all_targets = [], []\nfor X_batch, y_batch in parquet_batch_generator(parquet_files, image_col, label_col):\n    _, _, X_test, y_test = split_batch(X_batch, y_batch, train_fraction)\n    if len(X_test) == 0:\n        continue\n    preds = rf.predict(X_test)\n    all_preds.append(preds)\n    all_targets.append(y_test)\n\naccuracy = accuracy_score(np.concatenate(all_targets), np.concatenate(all_preds))\nprint(\"Random Forest Accuracy:\", accuracy)\n\n\nwith open(\"/kaggle/working/rf_model.pkl\", \"wb\") as f:\n    pickle.dump(rf, f)\ndel rf, X_rf, y_rf, rf_X_list, rf_y_list, all_preds, all_targets\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T06:27:31.899470Z","iopub.execute_input":"2025-12-10T06:27:31.899741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport numpy as np\nimport glob\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import accuracy_score\n\nparquet_files = [\"/kaggle/input/complete-patches-data-physionet-2025/patches_batch_67.parquet\"]\n\n\nbatch_size = 4000      \ntrain_fraction = 0.8   \nnum_epochs = 1         \n\nsvm = SGDClassifier(\n    loss=\"hinge\",      \n    learning_rate=\"optimal\",\n    eta0=0.01,\n    max_iter=1,\n    tol=None,\n    warm_start=True    \n)\n\nfirst_batch = True\n\ndef parquet_batch_generator(files, batch_size=1024):\n    for f in files:\n        pf = pq.ParquetFile(f)\n        for batch in pf.iter_batches(batch_size=batch_size):\n            df = batch.to_pandas()\n            X = np.stack(df[\"image_patch\"].values).astype(np.float32)\n            y = df[\"labelClf\"].to_numpy().astype(int)\n            yield X, y\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    print(f\"Epoch {epoch+1} starting…\")\n\n    for Xb, yb in parquet_batch_generator(parquet_files, batch_size):\n\n        # in-batch train/test split\n        m = np.random.rand(len(Xb)) < train_fraction\n        X_train, y_train = Xb[m], yb[m]\n\n        if len(X_train) == 0:\n            continue\n\n        if first_batch:\n            svm.partial_fit(X_train, y_train, classes=np.array([0,1]))\n            first_batch = False\n        else:\n            svm.partial_fit(X_train, y_train)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\njoblib.dump(svm, \"/kaggle/working/svm_waveform.pkl\")\nprint(\"Saved SVM model.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds = []\ntest_true  = []\n\nfor Xb, yb in parquet_batch_generator(parquet_files, batch_size):\n\n    m = np.random.rand(len(Xb)) >= train_fraction\n    X_test, y_test = Xb[m], yb[m]\n\n    if len(X_test) == 0:\n        continue\n\n    preds = svm.predict(X_test)\n    test_preds.append(preds)\n    test_true.append(y_test)\n\n# concatenate\ntest_preds = np.concatenate(test_preds)\ntest_true  = np.concatenate(test_true)\n\nacc = accuracy_score(test_true, test_preds)\nprint(\"Test Accuracy:\", acc)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport numpy as np\nimport glob\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport time\n\n# -------------------------------\n# Config\n# -------------------------------\nparquet_files = glob.glob(\"/kaggle/input/complete-patches-data-physionet-2025/*.parquet\")\n\nMAX_FILES_PER_EPOCH = 20        \nMAX_BATCHES_PER_FILE = 20       \nEPOCHS = 5                     \nTRAIN_FRACTION = 0.8\nBATCH_SIZE_PARQUET = 1024\nBATCH_SIZE_LOADER = 64\nH, W = 64, 64\nEARLY_STOP_PATIENCE = 3\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n\nclass PatchDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.tensor(\n            X.reshape(-1, 1, H, W), dtype=torch.float32\n        )\n        self.y = torch.tensor(y, dtype=torch.long)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx]\n\n\nclass CNN(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, 3, padding=1)\n        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)\n        self.fc1 = nn.Linear(32 * (H//4) * (W//4), 64)\n        self.fc2 = nn.Linear(64, num_classes)\n\n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.max_pool2d(x, 2)\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2)\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        return self.fc2(x)\n\n\ndef parquet_batch_generator(file_paths, max_files, max_batches, batch_size):\n    files = np.random.choice(file_paths, size=min(max_files, len(file_paths)), replace=False)\n    for f in files:\n        pf = pq.ParquetFile(f)\n        for i, batch in enumerate(pf.iter_batches(batch_size=batch_size)):\n            if i >= max_batches:\n                break\n            df_batch = batch.to_pandas()\n            X = np.stack(df_batch[\"image_patch\"].values)\n            y = df_batch[\"labelClf\"].to_numpy()\n            yield X, y\n\n\nfor Xb, yb in parquet_batch_generator(parquet_files, 1, 1, BATCH_SIZE_PARQUET):\n    num_classes = len(np.unique(yb))\n    break\n\ncnn = CNN(num_classes).to(device)\noptimizer = torch.optim.Adam(cnn.parameters(), lr=1e-3)\ncriterion = nn.CrossEntropyLoss()\n\n\nbest_loss = float(\"inf\")\npatience_counter = 0\n\nstart_time = time.time()\n\nfor epoch in range(EPOCHS):\n    cnn.train()\n    epoch_losses = []\n\n    for Xb, yb in parquet_batch_generator(\n        parquet_files,\n        MAX_FILES_PER_EPOCH,\n        MAX_BATCHES_PER_FILE,\n        BATCH_SIZE_PARQUET\n    ):\n        \n        mask = np.random.rand(len(Xb)) < TRAIN_FRACTION\n        X_train, y_train = Xb[mask], yb[mask]\n        if len(X_train) == 0:\n            continue\n\n        train_ds = PatchDataset(X_train, y_train)\n        train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE_LOADER, shuffle=True)\n\n        for X_batch, y_batch in train_loader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n\n            optimizer.zero_grad()\n            logits = cnn(X_batch)\n            loss = criterion(logits, y_batch)\n            loss.backward()\n            optimizer.step()\n\n            epoch_losses.append(loss.item())\n\n        # Force stop if exceeding 1 hour\n        if time.time() - start_time > 3600:\n            print(\"Time limit hit — stopping training early.\")\n            break\n\n    avg_loss = np.mean(epoch_losses) if epoch_losses else best_loss\n    print(f\"Epoch {epoch+1}/{EPOCHS} - Loss={avg_loss:.4f}\")\n\n    # Early stopping\n    if avg_loss < best_loss:\n        best_loss = avg_loss\n        patience_counter = 0\n        torch.save(cnn.state_dict(), \"/kaggle/working/cnn_best.pth\")\n    else:\n        patience_counter += 1\n        if patience_counter >= EARLY_STOP_PATIENCE:\n            print(\"Early stopping triggered.\")\n            break\n\n\ntorch.save(cnn.state_dict(), \"/kaggle/working/cnn_final.pth\")\nprint(\"Training completed.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport numpy as np\nimport glob\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# -------------------------------\n# Config\n# -------------------------------\nparquet_files = glob.glob(\"/kaggle/input/complete-patches-data-physionet-2025/*.parquet\")\nsubset_batches = 10        \nbatch_size = 1024          \nloader_batch_size = 128    \nH, W = 64, 64\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n\nclass PatchDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.tensor(X.reshape(-1, 1, H, W), dtype=torch.float32)\n        self.y = torch.tensor(y, dtype=torch.long)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx]\n\n\nclass CNN(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, 3, padding=1)\n        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)\n        self.fc1   = nn.Linear(32 * (H//4) * (W//4), 64)\n        self.fc2   = nn.Linear(64, num_classes)\n\n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.max_pool2d(x, 2)\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2)\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        return self.fc2(x)\n\n\nfor f in parquet_files:\n    pf = pq.ParquetFile(f)\n    batch = next(pf.iter_batches(batch_size=batch_size))\n    df_batch = batch.to_pandas()\n    num_classes = len(np.unique(df_batch['labelClf'].to_numpy()))\n    break\n\ncnn = CNN(num_classes).to(device)\ncnn.load_state_dict(torch.load(\"/kaggle/working/cnn_final.pth\", map_location=device))\ncnn.eval()\n\n\nall_preds = []\nall_labels = []\nbatches_processed = 0\n\nwith torch.no_grad():\n    for f in parquet_files:\n        pf = pq.ParquetFile(f)\n        for batch in pf.iter_batches(batch_size=batch_size):\n            df_batch = batch.to_pandas()\n            X = np.stack(df_batch['image_patch'].values)\n            y = df_batch['labelClf'].to_numpy()\n\n            eval_ds = PatchDataset(X, y)\n            eval_loader = DataLoader(eval_ds, batch_size=loader_batch_size, shuffle=False)\n\n            for Xb, yb in eval_loader:\n                Xb, yb = Xb.to(device), yb.to(device)\n                logits = cnn(Xb)\n                preds = torch.argmax(logits, dim=1)\n                all_preds.append(preds.cpu().numpy())\n                all_labels.append(yb.cpu().numpy())\n\n            batches_processed += 1\n            if batches_processed >= subset_batches:\n                break\n        if batches_processed >= subset_batches:\n            break\n\n# Combine results\nall_preds = np.concatenate(all_preds)\nall_labels = np.concatenate(all_labels)\n\naccuracy = np.mean(all_preds == all_labels)\nprint(f\"Quick evaluation accuracy on subset: {accuracy*100:.2f}%\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}