{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"sourceType":"competition"},{"sourceId":13746387,"sourceType":"datasetVersion","datasetId":8747012}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import cc3d\nexcept:\n    #https://pypi.org/project/connected-components-3d/\n    #!pip install connected-components-3d\n\n    !ls /kaggle/input/hengck23-submit-physionet/hengck23-submit-physionet/setup\n    !pip install connected-components-3d --no-index --find-links=file:///kaggle/input/hengck23-submit-physionet/hengck23-submit-physionet/setup\n\nimport cc3d\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport torch\nimport matplotlib.pyplot as plt\nimport matplotlib\n#matplotlib.use('TkAgg')\nimport shutil\n\nimport sys\nsys.path.append('/kaggle/input/hengck23-submit-physionet/hengck23-submit-physionet')\n\nprint('import ok!!!')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:25:35.901282Z","iopub.execute_input":"2025-11-15T20:25:35.901528Z","iopub.status.idle":"2025-11-15T20:25:44.475071Z","shell.execute_reply.started":"2025-11-15T20:25:35.901508Z","shell.execute_reply":"2025-11-15T20:25:44.474351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nMODE   = 'submit'  # submit  local fake\nDEVICE = 'cuda'\nFLOAT_TYPE = torch.float16 #torch.bfloat16\nFAIL_ID = []\n\nKAGGLE_DIR = \\\n\t'/kaggle/input/physionet-ecg-image-digitization'\nWEIGHT_DIR = \\\n\t'/kaggle/input/hengck23-submit-physionet/hengck23-submit-physionet/weight'\nOUT_DIR = \\\n    f'/kaggle/working/output-{MODE}'\n\ndef make_test_fake_df(): \n    valid_df = pd.read_csv(f'{KAGGLE_DIR}/train.csv')\n    valid_df.loc[:,'id']=valid_df['id'].astype(str) \n    fake_test_df=[]\n    for i,d in valid_df.iterrows():\n        #if i==4: break\n        image_id = d['id']\n    \n        truth_df = pd.read_csv(f'{KAGGLE_DIR}/train/{image_id}/{image_id}.csv')\n        non_nan_count = truth_df.count()\n        #print(i,image_id,non_nan_count)\n        #print(non_nan_count.index)\n    \n        #lead\tfs\tnumber_of_rows \n        this_df = pd.DataFrame({\n            'id':image_id ,\n            'lead':non_nan_count.index,\n            'fs': d['fs'],\n            'number_of_rows':non_nan_count.values \n        })\n        fake_test_df.append(this_df)\n        if i==0: print(this_df)\n    fake_test_df = pd.concat(fake_test_df)\n    return fake_test_df\n\n\n# set valid/test data\nif MODE == 'local':\n\tfrom sample_list import ERROR_ID\n\tvalid_df = pd.read_csv(f'{KAGGLE_DIR}/train.csv')\n\tvalid_df['id']=valid_df['id'].astype(str)\n\n\tvalid_id = [\n\t\tf'{image_id}-{type_id}' for image_id in ERROR_ID\n\t\t#f'{image_id}-{type_id}' for image_id in valid_df['id'].values[500:]\n\t\tfor type_id in ['0001', '0003', '0004', '0005', '0006', '0009', '0010', '0011', '0012']\n\t]\n\tvalid_id = [\n        '11842146-0012','144746082-0009','225208096-0006', '2289894144-0012','1617515072-0006',\n        '2289894144-0010','2566168201-0009', '2659677149-0011'\n    ]\n    \nif MODE == 'submit':\n\tvalid_df = pd.read_csv(f'{KAGGLE_DIR}/test.csv')\n\tvalid_df['id']=valid_df['id'].astype(str) \n\tvalid_id = valid_df['id'].unique().tolist()\n\nif MODE == 'fake':\n\tvalid_df = make_test_fake_df()\n\tvalid_df['id']=valid_df['id'].astype(str) \n\tvalid_id = valid_df['id'].unique().tolist()\n\n#--------------------------------------\n\ndef read_image(sample_id):\n    if MODE == 'local':\n        image_id, type_id = sample_id.split('-')\n        image = cv2.imread(f'{KAGGLE_DIR}/train/{image_id}/{image_id}-{type_id}.png', cv2.IMREAD_COLOR_RGB)\n        return image\n    if MODE == 'submit':\n        image_id = sample_id\n        image = cv2.imread(f'{KAGGLE_DIR}/test/{image_id}.png', cv2.IMREAD_COLOR_RGB)\n        return image\n    if MODE == 'fake':\n        image_id = sample_id \n        type_id = ['0001', '0003', '0004', '0005', '0006', '0009', '0010', '0011', '0012'][\n            int(image_id)%9\n        ] \n        image = cv2.imread(f'{KAGGLE_DIR}/train/{image_id}/{image_id}-{type_id}.png', cv2.IMREAD_COLOR_RGB)\n        return image\n\ndef read_sampling_length(sample_id):\n\tif MODE == 'local':\n\t\timage_id, type_id = sample_id.split('-')\n\t\td = valid_df[valid_df['id']==image_id].iloc[0]\n\t\tlength = d.sig_len\n\t\treturn length\n\tif MODE == 'submit':\n\t\timage_id = sample_id\n\t\td = valid_df[\n\t\t\t(valid_df['id']==image_id) & (valid_df['lead']=='II')\n\t\t].iloc[0]\n\t\tlength = d.number_of_rows\n\t\treturn length\n\tif MODE == 'fake':\n\t\timage_id = sample_id\n\t\td = valid_df[\n\t\t\t(valid_df['id']==image_id) & (valid_df['lead']=='II')\n\t\t].iloc[0]\n\t\tlength = d.fs*10  #d.number_of_rowsd.number_of_rows\n\t\treturn length\n\n#valid_id = valid_id[:300]\nprint('valid_id:', len(valid_id))\nprint('\\t', valid_id[:3], '...')\nprint('setting ok!!!\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:25:44.476359Z","iopub.execute_input":"2025-11-15T20:25:44.476754Z","iopub.status.idle":"2025-11-15T20:25:44.507602Z","shell.execute_reply.started":"2025-11-15T20:25:44.476661Z","shell.execute_reply":"2025-11-15T20:25:44.50701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# stage0\nprint('*** STARTING STAGE0 ***')\n\nfrom stage0_model import Net as Stage0Net\nfrom stage0_common import *\n\nos.makedirs(f'{OUT_DIR}/normalised', exist_ok=True)\n\ndef run_stage0():\n\tstage0_net = Stage0Net(pretrained=False)\n\tstage0_net = load_net(stage0_net, f'{WEIGHT_DIR}/stage0-last.checkpoint.pth')\n\tstage0_net.to(DEVICE)\n\n\tstart_timer = timer()\n\tfor n, sample_id in enumerate(valid_id):\n\t\ttimestamp = time_to_str(timer() - start_timer, 'sec')\n\t\tprint(f'\\r\\t {n:4d} {sample_id}', timestamp, end='', flush=True)\n\n\t\timage = read_image(sample_id)\n\t\tbatch = image_to_batch(image)\n\n\t\twith torch.amp.autocast('cuda', dtype=FLOAT_TYPE):\n\t\t\twith torch.no_grad():\n\t\t\t\toutput = stage0_net(batch)\n\n\t\t\t\ttry:\n\t\t\t\t\trotated, keypoint = output_to_predict(image, batch, output)\n\t\t\t\t\tnormalised, keypoint, homo = normalise_by_homography(rotated, keypoint)\n\t\t\t\t\t# ---\n\t\t\t\t\tcv2.imwrite(f'{OUT_DIR}/normalised/{sample_id}.norm.png', cv2.cvtColor(normalised, cv2.COLOR_RGB2BGR))\n\t\t\t\t\tnp.save(f'{OUT_DIR}/normalised/{sample_id}.homo.npy', homo)\n\t\t\t\texcept:\n\t\t\t\t\tFAIL_ID.append(sample_id)\n\n\t\ttorch.cuda.empty_cache()\n\t\tif n<10: # optional: show results\n\t\t\toverlay = draw_results_stage0(rotated, keypoint)\n\t\t\tprint('')\n\t\t\tprint('demo results for stage0--------------')\n\t\t\tprint(sample_id)\n\t\t\tplt.imshow(image);plt.show()\n\t\t\tplt.imshow(overlay);plt.show()\n\t\t\tplt.imshow(normalised);plt.show()\n\t\t\t\n\tprint('')\n\nrun_stage0()\nprint('run_stage0() ok!!!\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:25:44.508308Z","iopub.execute_input":"2025-11-15T20:25:44.508547Z","iopub.status.idle":"2025-11-15T20:25:57.957921Z","shell.execute_reply.started":"2025-11-15T20:25:44.508521Z","shell.execute_reply":"2025-11-15T20:25:57.957233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# stage1\nprint('*** STARTING STAGE1 ***')\n\nfrom stage1_model import Net as Stage1Net\nfrom stage1_common import *\n\nos.makedirs(f'{OUT_DIR}/rectified', exist_ok=True)\n\ndef run_stage1():\n\tstage1_net = Stage1Net(pretrained=False)\n\tstage1_net = load_net(stage1_net, f'{WEIGHT_DIR}/stage1-last.checkpoint.pth')\n\tstage1_net.to(DEVICE)\n\n\tstart_timer = timer()\n\tfor n, sample_id in enumerate(valid_id):\n\t\ttimestamp = time_to_str(timer() - start_timer, 'sec')\n\t\tprint(f'\\r\\t {n:4d} {sample_id}', timestamp, end='', flush=True)\n\t\tif sample_id in FAIL_ID: continue\n\n\t\timage = cv2.imread(f'{OUT_DIR}/normalised/{sample_id}.norm.png', cv2.IMREAD_COLOR_RGB)\n\t\tbatch = {\n\t\t\t'image': torch.from_numpy(np.ascontiguousarray(image.transpose(2, 0, 1))).unsqueeze(0),\n\t\t}\n\t\tnum_tta = 1\n\n\t\twith torch.amp.autocast('cuda', dtype=FLOAT_TYPE): #torch.bfloat16\n\t\t\twith torch.no_grad():\n\t\t\t\toutput = stage1_net(batch)\n\n\t\t\t\ttry:\n\t\t\t\t\tgridpoint_xy, more = output_to_predict(image, batch, output)\n\t\t\t\t\trectified = rectify_image(image, gridpoint_xy)\n\t\t\t\t\t# ---\n\t\t\t\t\tcv2.imwrite(f'{OUT_DIR}/rectified/{sample_id}.rect.png', cv2.cvtColor(rectified, cv2.COLOR_RGB2BGR))\n\t\t\t\t\tnp.save(f'{OUT_DIR}/rectified/{sample_id}.gridpoint_xy.npy',gridpoint_xy)\n\t\t\t\texcept:\n\t\t\t\t\tFAIL_ID.append(sample_id)\n\n\t\ttorch.cuda.empty_cache()\n\t\tif n<10: # optional: show results\n\t\t\toverlay = draw_mapping(image, gridpoint_xy) #\n\t\t\tghfiltered, gvfiltered = draw_results_stage1(more)\n            \n\t\t\t\n\t\t\tprint('')\n\t\t\tprint('demo results for stage1--------------')\n\t\t\tprint(sample_id)\n\t\t\tplt.imshow(overlay);plt.show()\n\t\t\tplt.imshow(gvfiltered);plt.show()\n\t\t\tplt.imshow(ghfiltered);plt.show()\n\t\t\tplt.imshow(rectified);plt.show()\n             \n\tprint('')\n\nrun_stage1()\nprint('FAIL_ID:', FAIL_ID)\nprint('run_stage1() ok!!!\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:25:57.959278Z","iopub.execute_input":"2025-11-15T20:25:57.9595Z","iopub.status.idle":"2025-11-15T20:26:07.062213Z","shell.execute_reply.started":"2025-11-15T20:25:57.959481Z","shell.execute_reply":"2025-11-15T20:26:07.061558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# stage2\nprint('*** STARTING STAGE2 ***')\n\nfrom stage2_model import Net as Stage2Net, prob_to_series_by_max\nfrom stage2_common import *\n\nos.makedirs(f'{OUT_DIR}/digitalised', exist_ok=True)\n#os.makedirs(f'{OUT_DIR}/debug', exist_ok=True)\n\ndef run_stage2():\n\tstage2_net = Stage2Net(pretrained=False)\n\tstage2_net = load_net(\n\t\tstage2_net,\n\t\tf'{WEIGHT_DIR}/stage2-00005810.checkpoint.pth'\n\t)\n\tstage2_net.to(DEVICE)\n\n\tstart_timer = timer()\n\tfor n, sample_id in enumerate(valid_id):\n\t\t# sample_id =\\\n\t\t# \t'1445349505-0006' #'1617515072-0006'\n\n\t\ttimestamp = time_to_str(timer() - start_timer, 'sec')\n\t\tprint(f'\\r\\t {n:4d} {sample_id}', timestamp, end='', flush=True)\n\t\tif sample_id in FAIL_ID: continue\n\n\t\timage = cv2.imread(f'{OUT_DIR}/rectified/{sample_id}.rect.png', cv2.IMREAD_COLOR_RGB)\n\t\tlength = read_sampling_length(sample_id) #5120\n\n\t\t# at rectified coord frame: H, W = 1700, 2200\n\t\tx0, x1 = 0, 2176\n\t\ty0, y1 = 0, 1696\n\t\tzero_mv = [ 703.5, 987.5, 1271.5, 1531.5 ]\n\t\tmv_to_pixel = 79.0\n\t\tt0,t1 = timespan = 118, 2080\n\n\t\tcrop = image[y0:y1, x0:x1]\n\t\tbatch = {\n\t\t\t'image': torch.from_numpy(np.ascontiguousarray(crop.transpose(2, 0, 1))).unsqueeze(0),\n\t\t}\n\t\twith torch.amp.autocast('cuda', dtype=FLOAT_TYPE):\n\t\t\twith torch.no_grad():\n\t\t\t\toutput = stage2_net(batch)\n\n\t\t#---\n\t\ttry:\n\t\t#if 1:\n\t\t\tpixel = output['pixel'].float().data.cpu().numpy()[0]\n\t\t\tseries_in_pixel = pixel_to_series(pixel[..., t0:t1], zero_mv, length)\n\t\t\tseries = (np.array(zero_mv).reshape(4, 1) - series_in_pixel) / mv_to_pixel\n\t\t\tseries = filter_series_by_limits(series)\n\n\t\t\t# ---\n\t\t\t#cv2.imwrite(f'{OUT_DIR}/digitalised/{sample_id}.lead.png', cv2.cvtColor(overlay, cv2.COLOR_RGB2BGR))\n\t\t\tnp.save(f'{OUT_DIR}/digitalised/{sample_id}.series.npy', series)\n\n\t\texcept:\n\t\t\tFAIL_ID.append(sample_id)\n\n\t\tif n<10: # optional: show results\n\t\t\toverlay = draw_lead_pixel(crop, pixel)\n\t\t\tplt.imshow(overlay); plt.show()\n\t \n\t\t\tif MODE=='local':\n\t\t\t\ttruth_df = read_truth_series(sample_id,KAGGLE_DIR)\n\t\t\t\ttruth_series = truth_df[['series0','series1','series2','series3',]].values.T\n\n\t\t\tt = np.arange(len(series[0]))\n\t\t\tfig, axes = plt.subplots(4, 1, figsize=(12, 10))\n\t\t\tfor j in range(4):\n\t\t\t\tsnr=0\n\t\t\t\taxes[j].plot(t, series[j], alpha=1.0, color='blue', linewidth=1, label='predict')\n\t\t\t\tif MODE=='local':\n\t\t\t\t\taxes[j].plot(t, truth_series[j], alpha=0.5, color='red', linewidth=1,label='truth')\n\t\t\t\t\tsnr = -np_snr(series[j], truth_series[j])\n\n\t\t\t\taxes[j].set_title(f'snr {snr:8.3f}')\n\t\t\t\taxes[j].legend()\n\t\t\tplt.show()\n\tprint('')\n\nrun_stage2()\nprint('FAIL_ID:', FAIL_ID)\nprint('run_stage2() ok!!!\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:26:07.062941Z","iopub.execute_input":"2025-11-15T20:26:07.063308Z","iopub.status.idle":"2025-11-15T20:26:12.871816Z","shell.execute_reply.started":"2025-11-15T20:26:07.063291Z","shell.execute_reply":"2025-11-15T20:26:12.871112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# make submission csv\n# FAIL_ID = [1053922973, ]\ndef make_submission():\n    print('===========================================')\n    print('making submission csv ...')\n\n    submit_df = []\n    gb = valid_df.groupby('id')\n    for rec_idx, (sample_id, df) in enumerate(gb):\n\n        # if sample_id in FAIL_ID:\n        #     series_by_lead = {}\n        #     for _, d in df.iterrows():\n        #         series_by_lead[d.lead] = np.zeros(d.number_of_rows)\n\n        try:\n            series = np.load(f'{OUT_DIR}/digitalised/{sample_id}.series.npy')\n            _4_, L = series.shape\n\n            # https://www.kaggle.com/competitions/physionet-ecg-image-digitization/discussion/613179#3306701\n            # may be even or odd????\n            series_by_lead = {}\n            for l in range(3):\n                lead_names = [\n                    ['I',   'aVR', 'V1', 'V4'],\n                    ['II',  'aVL', 'V2', 'V5'],\n                    ['III', 'aVF', 'V3', 'V6'],\n                ][l]\n\n                index = [\n                    int(round(1 * L / 4)),\n                    int(round(2 * L / 4)),\n                    int(round(3 * L / 4)),\n                ]\n                split = np.split(series[l], index)\n                for k, s in zip(lead_names, split):\n                    series_by_lead[k] = s\n\n            # override II with long rhythm\n            series_by_lead['II'] = series[3]\n\n        except Exception as e:\n            # if anything goes wrong, fall back to zeros per lead\n            series_by_lead = {}\n            for _, d in df.iterrows():\n                series_by_lead[d.lead] = np.zeros(d.number_of_rows, dtype=np.float32)\n\n        # build rows for this sample_id\n        for _, d in df.iterrows():\n            s = series_by_lead[d.lead]\n            target_len = int(d.number_of_rows)\n\n            # Length correction by interpolation instead of duplicate+zeros\n            if len(s) != target_len:\n                x_old = np.linspace(0.0, 1.0, len(s), endpoint=False)\n                x_new = np.linspace(0.0, 1.0, target_len, endpoint=False)\n                s = np.interp(x_new, x_old, s)\n\n            # convert dtype\n            s = s.astype(np.float32)\n\n            # -------------------------------------------------\n            # ✅ ADD SMALL SAFE GAUSSIAN NOISE (leaderboard safe)\n            # -------------------------------------------------\n            noise_scale = 0.0015 * (np.std(s) + 1e-6)   # tiny 0.15% std\n            noise = np.random.normal(0, noise_scale, size=s.shape).astype(np.float32)\n            s = s + noise\n            s = np.clip(s, -5, 5)  # safe clipping\n            # -------------------------------------------------\n\n            series_by_lead[d.lead] = s\n\n            assert len(s) == target_len\n            print(f'\\r\\t {rec_idx} {sample_id} : {d.lead}', end='', flush=True)\n\n            row_id = [f'{sample_id}_{t}_{d.lead}' for t in range(target_len)]\n            this_df = pd.DataFrame({\n                'id': row_id,\n                'value': s,\n            })\n            submit_df.append(this_df)\n\n    print('')\n    submit_df = pd.concat(submit_df, axis=0, ignore_index=True, sort=False, copy=False)\n    print(submit_df.head())\n    submit_df.to_csv('submission.csv', index=False)\n\n\nif (MODE == 'fake') or (MODE == 'submit'):\n    make_submission()\n    print('make_submission() ok!!!\\n')\n    if MODE == 'submit':\n        shutil.rmtree(OUT_DIR)\n    !ls\n    # !rm -rf {OUT_DIR}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T20:26:12.872586Z","iopub.execute_input":"2025-11-15T20:26:12.872891Z","iopub.status.idle":"2025-11-15T20:26:13.536597Z","shell.execute_reply.started":"2025-11-15T20:26:12.872872Z","shell.execute_reply":"2025-11-15T20:26:13.535561Z"}},"outputs":[],"execution_count":null}]}