{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"machine_shape":"hm","gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":35273,"databundleVersionId":3351394,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmarking EchoClip zero-shot ejection fraction prediction\n\nEchoClip: https://github.com/echonet/echo_CLIP/tree/main  \nDataset: https://www.kaggle.com/competitions/echo2022","metadata":{}},{"cell_type":"markdown","source":"## Load / install packages","metadata":{}},{"cell_type":"code","source":"!pip install open-clip-torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T22:17:38.441381Z","iopub.execute_input":"2025-12-08T22:17:38.442257Z","iopub.status.idle":"2025-12-08T22:18:50.614812Z","shell.execute_reply.started":"2025-12-08T22:17:38.442229Z","shell.execute_reply":"2025-12-08T22:18:50.613894Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import open_clip\nimport torch\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom einops import repeat\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport torch.nn.functional as F\n\nfrom torchvision import transforms as T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:02:47.030337Z","iopub.execute_input":"2025-12-08T23:02:47.031062Z","iopub.status.idle":"2025-12-08T23:02:47.034821Z","shell.execute_reply.started":"2025-12-08T23:02:47.031035Z","shell.execute_reply":"2025-12-08T23:02:47.034198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helper functions","metadata":{}},{"cell_type":"code","source":"def load_and_process_video(path):\n    \"\"\"Custom pre-processing based on Echo2022 dataset.\"\"\"\n\n    transform = T.Compose([\n        T.Resize(size=(358, 224)),\n        T.CenterCrop(224),\n        T.Normalize(mean=0.5, std=0.25)\n    ])\n    \n    video = np.load(path) / 255\n       \n    # Model expects input to have 3 channels C.\n    video = repeat(video, \"Frames Height Width -> Frames C Height Width\", C=3)\n    \n    video = torch.from_numpy(video)\n    video = transform(video)\n    \n    return video.to(torch.bfloat16)\n\n@torch.no_grad()\ndef generate_video_embeddings(video, model) -> torch.Tensor:\n    model.eval()\n    video = video.cuda()\n    \n    # Be sure to normalize the CLIP embedding after calculating it to make\n    # cosine similarity between embeddings easier to calculate.\n    embedding = F.normalize(model.encode_image(video), dim=-1)\n    return embedding.unsqueeze(0)\n\n@torch.no_grad()\ndef generate_ef_prompt_embedding(model, tokenizer) -> tuple[torch.Tensor, list[int]]:\n    \"\"\"Ejection Fraction (EF) prompt embedding for EchoClip.\"\"\"\n    model.eval()\n    \n    # prompt taken from: https://github.com/echonet/echo_CLIP/blob/main/utils.py\n    ejection_fraction_prompts = [\n            \"THE LEFT VENTRICULAR EJECTION FRACTION IS ESTIMATED TO BE <#>% \",\n            \"LV EJECTION FRACTION IS <#>%. \",\n        ]\n    \n    prompts = []\n    prompt_values = []\n    \n    for prompt in ejection_fraction_prompts:\n        for i in range(101):\n            prompts.append(prompt.replace(\"<#>\", str(i)))\n            prompt_values.append(i)\n    \n    tokens = tokenizer(prompts).cuda()\n    return F.normalize(model.encode_text(tokens), dim=-1), prompt_values\n\ndef compute_regression_metric(\n    video_embeddings: torch.Tensor,\n    prompt_embeddings: torch.Tensor,\n    prompt_values: torch.Tensor,\n):\n    \"\"\"See: https://github.com/echonet/echo_CLIP/blob/main/utils.py#L82\"\"\"\n    per_frame_similarities = (\n        video_embeddings @ prompt_embeddings.T\n    )  # (N x Frames x Candidates)\n\n    # Sort the candidates by their similarity to the video\n    ranked_candidate_phrase_indices = torch.argsort(\n        per_frame_similarities, dim=-1, descending=True\n    )\n\n    # Convert matrix of indices to their corresponding continuous values.\n    prompt_values = torch.tensor(\n        prompt_values, device=video_embeddings.device\n    )  # (N x Frames x Candidates)\n    all_frames_ranked_values = prompt_values[ranked_candidate_phrase_indices]\n\n    # Taking the mean along dim=1 collapses the frames dimension\n    avg_frame_ranked_values = all_frames_ranked_values.float().mean(\n        dim=1\n    )  # (N x Candidates)\n\n    # The median of only the top 20% of predicted values is taken\n    # as the final predicted value\n    twenty_percent = int(avg_frame_ranked_values.shape[1] * 0.2)\n    final_prediction = avg_frame_ranked_values[:, :twenty_percent].median(dim=-1)[0]\n\n    return final_prediction\n\n\n@torch.no_grad()\ndef predict_ejection_fraction(video, model, tokenizer=None, prompt_embeddings=None, prompt_values=None) -> float:\n\n    video_embeddings = generate_video_embeddings(video, model)\n    \n    if tokenizer:\n        prompt_embeddings, prompt_values = generate_ef_prompt_embedding(model, tokenizer)\n    else:\n        assert prompt_embeddings is not None\n        assert prompt_values is not None\n\n    return compute_regression_metric(video_embeddings, prompt_embeddings, prompt_values).item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:42:30.899495Z","iopub.execute_input":"2025-12-08T23:42:30.899762Z","iopub.status.idle":"2025-12-08T23:42:30.909962Z","shell.execute_reply.started":"2025-12-08T23:42:30.899746Z","shell.execute_reply":"2025-12-08T23:42:30.909120Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load model","metadata":{}},{"cell_type":"code","source":"# See https://github.com/echonet/echo_CLIP/blob/main/zero_shot_example.py\n\nmodel, preprocess_train, preprocess_val = open_clip.create_model_and_transforms(\n    'hf-hub:mkaichristensen/echo-clip',\n    precision=\"bf16\",\n    device=\"cuda\"\n)\ntokenizer = open_clip.get_tokenizer('hf-hub:mkaichristensen/echo-clip')\n\nmodel = model.to(\"cuda\")\nmodel.eval();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:25:47.139091Z","iopub.execute_input":"2025-12-08T23:25:47.139404Z","iopub.status.idle":"2025-12-08T23:25:50.428018Z","shell.execute_reply.started":"2025-12-08T23:25:47.139382Z","shell.execute_reply":"2025-12-08T23:25:50.427431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/echo2022/train_data.csv\")\ntrain_df[\"prediction\"] = -1\n\nch4_path = \"/kaggle/input/echo2022/train_data/train_data/4CH/{}_4CH_sequence.npy\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T00:21:12.337802Z","iopub.execute_input":"2025-12-09T00:21:12.338096Z","iopub.status.idle":"2025-12-09T00:21:12.350906Z","shell.execute_reply.started":"2025-12-09T00:21:12.338076Z","shell.execute_reply":"2025-12-09T00:21:12.350215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Run predictions","metadata":{}},{"cell_type":"code","source":"preds_4ch = []\nprompt_embeddings, prompt_values = generate_ef_prompt_embedding(model, tokenizer)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:44:33.066738Z","iopub.execute_input":"2025-12-08T23:44:33.067027Z","iopub.status.idle":"2025-12-08T23:44:33.108442Z","shell.execute_reply.started":"2025-12-08T23:44:33.066982Z","shell.execute_reply":"2025-12-08T23:44:33.107876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for idx, row in tqdm(train_df.iterrows(), total=len(train_df)):\n    input_path = ch4_path.format(row.Patient_number)\n\n    input_video = load_and_process_video(input_path)\n    \n    prediction = predict_ejection_fraction(input_video, model,\n                                           prompt_embeddings=prompt_embeddings,\n                                           prompt_values=prompt_values)\n\n    train_df.loc[idx, \"prediction\"] = prediction","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:45:06.724776Z","iopub.execute_input":"2025-12-08T23:45:06.725264Z","iopub.status.idle":"2025-12-08T23:47:58.428190Z","shell.execute_reply.started":"2025-12-08T23:45:06.725238Z","shell.execute_reply":"2025-12-08T23:47:58.427417Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Results","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import r2_score, mean_absolute_error\nfrom scipy.stats import pearsonr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T23:59:37.260628Z","iopub.execute_input":"2025-12-08T23:59:37.261268Z","iopub.status.idle":"2025-12-08T23:59:37.264830Z","shell.execute_reply.started":"2025-12-08T23:59:37.261241Z","shell.execute_reply":"2025-12-08T23:59:37.264122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mae = mean_absolute_error(train_df.LV_ef, train_df.prediction)\nr2 = r2_score(train_df.LV_ef, train_df.prediction)\npearson = pearsonr(train_df.LV_ef, train_df.prediction)\n\nplt.plot([10, 90], [10, 90], color=\"gray\", ls=\"--\", zorder=0)\nplt.scatter(train_df.LV_ef, train_df.prediction, alpha=0.8, s=70, linewidths=1, edgecolors=\"white\", zorder=1)\n\nplt.annotate(f\"MAE={mae:.1f}\", (65, 30), color=\"red\")\nplt.annotate(f\"$R^2$={r2:.3f}\", (65, 25), color=\"red\")\nplt.annotate(\"Pearson:\", (65, 20), color=\"red\")\nplt.annotate(f\"    -> corr coef={pearson.statistic:.3f}\", (65, 15), color=\"red\")\nplt.annotate(f\"    -> p-value={pearson.pvalue:.3f}\", (65, 10), color=\"red\")\n\nplt.xlabel(\"Ground Truth\")\nplt.ylabel(\"Prediction\")\nplt.title(\"EchoCLIP zero-shot EF prediction\")\nplt.grid()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T00:05:37.654674Z","iopub.execute_input":"2025-12-09T00:05:37.654972Z","iopub.status.idle":"2025-12-09T00:05:37.873269Z","shell.execute_reply.started":"2025-12-09T00:05:37.654951Z","shell.execute_reply":"2025-12-09T00:05:37.872661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}