{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":64733,"databundleVersionId":8171035,"sourceType":"competition"},{"sourceId":65626,"databundleVersionId":8046133,"sourceType":"competition"},{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":71885,"databundleVersionId":8143495,"sourceType":"competition"},{"sourceId":40919,"sourceType":"modelInstanceVersion","modelInstanceId":34432,"modelId":48428}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<center><img src='https://media.licdn.com/dms/image/D5612AQEFNAog5YvQGA/article-cover_image-shrink_720_1280/0/1701862732849?e=1720051200&v=beta&t=5uNFKqQTfWt0CSUKxvAPOUDKviP1B1YyBmoL22VduDQ' width=\"768\"></center>\n\n<h1><center>[How to] Inference MLLM(Multimodal LLM) - LLaVA-Phi3 </center><h1>","metadata":{}},{"cell_type":"markdown","source":"# About this notebook\n- MLLM (Multimoda LLM) inference starter code\n  (Base is LLaVA + Phi3 [here](https://github.com/mbzuai-oryx/LLaVA-pp/).)\n- Describing images (It is also possible to predict audio by replacing the vision tower, but we are only dealing with images here.)\n- Visual Understanding (e.g. bbox)\n\n\nI want to show how to use `MLLM (Multimoda LLM)` inference in kaggle nodebook.\n\n\n\nIf this notebook is helpful, feel free to upvote :)\n\nAnd please upvote the original notebook and discussion as well.","metadata":{}},{"cell_type":"markdown","source":"`V1` - Run test and add bbox\n- LLM: Phi3 (LLaVA 7B is failed due to GPU OOM)\n\nPlease see,\n- https://llava-vl.github.io/\n- https://www.reddit.com/r/ChatGPT/comments/1cb16cg/microsoft_introduces_phi3_llm_that_runs_on_the/?rdt=57581\n- https://github.com/mbzuai-oryx/LLaVA-pp","metadata":{}},{"cell_type":"markdown","source":"<img src='https://llava-vl.github.io/images/llava_arch.png' width=\"768\">\n<img src='https://llava-vl.github.io/images/cmp_ironing.png' width=\"768\">","metadata":{}},{"cell_type":"markdown","source":"# Install","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/mbzuai-oryx/LLaVA-pp.git","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:04:01.18462Z","iopub.execute_input":"2024-04-30T15:04:01.185303Z","iopub.status.idle":"2024-04-30T15:04:03.447676Z","shell.execute_reply.started":"2024-04-30T15:04:01.185259Z","shell.execute_reply":"2024-04-30T15:04:03.446516Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true,"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cd LLaVA-pp","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:04:03.450061Z","iopub.execute_input":"2024-04-30T15:04:03.450443Z","iopub.status.idle":"2024-04-30T15:04:03.45694Z","shell.execute_reply.started":"2024-04-30T15:04:03.450407Z","shell.execute_reply":"2024-04-30T15:04:03.456132Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!git submodule update --init --recursive","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:04:03.458101Z","iopub.execute_input":"2024-04-30T15:04:03.458366Z","iopub.status.idle":"2024-04-30T15:04:05.696276Z","shell.execute_reply.started":"2024-04-30T15:04:03.458344Z","shell.execute_reply":"2024-04-30T15:04:05.695105Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:04:33.798795Z","iopub.execute_input":"2024-04-30T15:04:33.799268Z","iopub.status.idle":"2024-04-30T15:04:34.738483Z","shell.execute_reply.started":"2024-04-30T15:04:33.799236Z","shell.execute_reply":"2024-04-30T15:04:34.737536Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cp Phi-3-V/train.py LLaVA/llava/train/train.py\n!cp Phi-3-V/llava_phi3.py LLaVA/llava/model/language_model/llava_phi3.py\n!cp Phi-3-V/builder.py LLaVA/llava/model/builder.py\n!cp Phi-3-V/model__init__.py LLaVA/llava/model/__init__.py\n!cp Phi-3-V/main__init__.py LLaVA/llava/__init__.py\n!cp Phi-3-V/conversation.py LLaVA/llava/conversation.py","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:04:40.37185Z","iopub.execute_input":"2024-04-30T15:04:40.372498Z","iopub.status.idle":"2024-04-30T15:04:45.957558Z","shell.execute_reply.started":"2024-04-30T15:04:40.37246Z","shell.execute_reply":"2024-04-30T15:04:45.956261Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd LLaVA\n!pip install --upgrade pip  > /dev/null\n!pip install -e .  > /dev/null\n!pip install git+https://github.com/huggingface/transformers@a98c41798cf6ed99e1ff17e3792d6e06a2ff2ff3  > /dev/null\n\n!export PYTHONPATH=\"./:$PYTHONPATH\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:07:05.741982Z","iopub.execute_input":"2024-04-30T15:07:05.742822Z","iopub.status.idle":"2024-04-30T15:09:19.01908Z","shell.execute_reply.started":"2024-04-30T15:07:05.742773Z","shell.execute_reply":"2024-04-30T15:09:19.01788Z"},"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import required packages","metadata":{}},{"cell_type":"code","source":"import torch\nfrom llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IMAGE_PLACEHOLDER\nfrom llava.conversation import conv_templates\nfrom llava.model.builder import load_pretrained_model\nfrom llava.utils import disable_torch_init\nfrom llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path\nimport requests\nfrom PIL import Image\nfrom io import BytesIO\nimport re\nfrom llava.utils import disable_torch_init","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:09:30.839468Z","iopub.execute_input":"2024-04-30T15:09:30.839868Z","iopub.status.idle":"2024-04-30T15:09:46.928488Z","shell.execute_reply.started":"2024-04-30T15:09:30.839833Z","shell.execute_reply":"2024-04-30T15:09:46.927491Z"},"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define helper functions","metadata":{}},{"cell_type":"code","source":"def image_parser(args):\n    out = args.image_file.split(args.sep)\n    return out\n\n\ndef load_image(image_file):\n    if image_file.startswith(\"http\") or image_file.startswith(\"https\"):\n        response = requests.get(image_file)\n        image = Image.open(BytesIO(response.content)).convert(\"RGB\")\n    else:\n        image = Image.open(image_file).convert(\"RGB\")\n    return image\n\n\ndef load_images(image_files):\n    out = []\n    for image_file in image_files:\n        image = load_image(image_file)\n        out.append(image)\n    return out","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:15:49.115008Z","iopub.execute_input":"2024-04-30T15:15:49.115505Z","iopub.status.idle":"2024-04-30T15:15:49.124248Z","shell.execute_reply.started":"2024-04-30T15:15:49.115469Z","shell.execute_reply":"2024-04-30T15:15:49.123211Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load LLaVA-Phi3","metadata":{}},{"cell_type":"code","source":"disable_torch_init()\n\nmodel_path = \"/kaggle/input/llava-phi-3-mini-4k-instruct/transformers/1.0.0/1/llava-phi-3-mini-4k-instruct\"\nmodel_name = get_model_name_from_path(model_path)\ntokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:09:56.279526Z","iopub.execute_input":"2024-04-30T15:09:56.279895Z","iopub.status.idle":"2024-04-30T15:11:20.816778Z","shell.execute_reply.started":"2024-04-30T15:09:56.279868Z","shell.execute_reply":"2024-04-30T15:11:20.815986Z"},"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create the prompt in Phi-3 Format","metadata":{}},{"cell_type":"markdown","source":"DEFAULT_IM_START_TOKEN\n- \\<im_start\\>\n\nDEFAULT_IMAGE_TOKEN\n- \\<image\\>\n\nDEFAULT_IM_END_TOKEN\n- \\<im_end\\>","metadata":{}},{"cell_type":"code","source":"qs = \"Please describe the plant traits in detail. Please think step by step.\"\n \nconv_mode = \"phi3_instruct\"\n\nimage_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN\nif IMAGE_PLACEHOLDER in qs:\n    if model.config.mm_use_im_start_end:\n        qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs)\n    else:\n        qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs)\nelse:\n    if model.config.mm_use_im_start_end:\n        qs = image_token_se + \"\\n\" + qs\n    else:\n        qs = DEFAULT_IMAGE_TOKEN + \"\\n\" + qs\n\nconv = conv_templates[conv_mode].copy()\nconv.system = \"\"\"<|system|>\\nYou are a helpful visual AI assistant.\"\"\"\nconv.append_message(conv.roles[0], qs)\nconv.append_message(conv.roles[1], None)\nprompt = conv.get_prompt()\n\nprint(prompt)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:13:17.94713Z","iopub.execute_input":"2024-04-30T15:13:17.947547Z","iopub.status.idle":"2024-04-30T15:13:17.955984Z","shell.execute_reply.started":"2024-04-30T15:13:17.947516Z","shell.execute_reply":"2024-04-30T15:13:17.95508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference for plant iamge - Describing plants","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nsaple_img_path = '/kaggle/input/planttraits2024/train_images/100438030.jpeg'\n\nimg = mpimg.imread(saple_img_path)\nplt.imshow(img)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:14:26.46729Z","iopub.execute_input":"2024-04-30T15:14:26.468352Z","iopub.status.idle":"2024-04-30T15:14:26.737604Z","shell.execute_reply.started":"2024-04-30T15:14:26.468307Z","shell.execute_reply":"2024-04-30T15:14:26.736731Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_name = saple_img_path\nimage_files = [image_name]\nimages = load_images(image_files)\nimage_sizes = [x.size for x in images]\nimages_tensor = process_images(\n    images,\n    image_processor,\n    model.config\n).to(model.device, dtype=torch.float16)\n\ninput_ids = (\n    tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\")\n    .unsqueeze(0)\n    .cuda()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:16:01.120014Z","iopub.execute_input":"2024-04-30T15:16:01.120959Z","iopub.status.idle":"2024-04-30T15:16:01.159112Z","shell.execute_reply.started":"2024-04-30T15:16:01.120924Z","shell.execute_reply":"2024-04-30T15:16:01.158271Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temperature = 0.2\ntop_p = 0.7\nnum_beams = 1\nmax_new_tokens = 512\n\nwith torch.inference_mode():\n    output_ids = model.generate(\n        input_ids,\n        images=images_tensor,\n        image_sizes=image_sizes,\n        do_sample=True if temperature > 0 else False,\n        temperature=temperature,\n        top_p=top_p,\n        num_beams=num_beams,\n        max_new_tokens=max_new_tokens,\n        use_cache=True,\n    )\n\noutputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()\noutputs = outputs.replace(\"<|end|>\", \"\").strip()\n#print(f\"\\n{outputs}\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:16:08.95189Z","iopub.execute_input":"2024-04-30T15:16:08.952543Z","iopub.status.idle":"2024-04-30T15:16:14.633754Z","shell.execute_reply.started":"2024-04-30T15:16:08.95251Z","shell.execute_reply":"2024-04-30T15:16:14.632837Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:16:14.635485Z","iopub.execute_input":"2024-04-30T15:16:14.63591Z","iopub.status.idle":"2024-04-30T15:16:14.641654Z","shell.execute_reply.started":"2024-04-30T15:16:14.635878Z","shell.execute_reply":"2024-04-30T15:16:14.64084Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference other images","metadata":{}},{"cell_type":"code","source":"saple_img_path = '/kaggle/input/planttraits2024/train_images/100282707.jpeg'\n\nimg = mpimg.imread(saple_img_path)\nplt.imshow(img)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:17:09.001985Z","iopub.execute_input":"2024-04-30T15:17:09.002842Z","iopub.status.idle":"2024-04-30T15:17:09.241889Z","shell.execute_reply.started":"2024-04-30T15:17:09.002812Z","shell.execute_reply":"2024-04-30T15:17:09.24089Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_name = saple_img_path\nimage_files = [image_name]\nimages = load_images(image_files)\nimage_sizes = [x.size for x in images]\nimages_tensor = process_images(\n    images,\n    image_processor,\n    model.config\n).to(model.device, dtype=torch.float16)\n\ninput_ids = (\n    tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\")\n    .unsqueeze(0)\n    .cuda()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:17:12.274876Z","iopub.execute_input":"2024-04-30T15:17:12.275532Z","iopub.status.idle":"2024-04-30T15:17:12.296616Z","shell.execute_reply.started":"2024-04-30T15:17:12.275499Z","shell.execute_reply":"2024-04-30T15:17:12.295811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temperature = 0.2\ntop_p = 0.7\nnum_beams = 1\nmax_new_tokens = 512\n\nwith torch.inference_mode():\n    output_ids = model.generate(\n        input_ids,\n        images=images_tensor,\n        image_sizes=image_sizes,\n        do_sample=True if temperature > 0 else False,\n        temperature=temperature,\n        top_p=top_p,\n        num_beams=num_beams,\n        max_new_tokens=max_new_tokens,\n        use_cache=True,\n    )\n\noutputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()\noutputs = outputs.replace(\"<|end|>\", \"\").strip()\n#print(f\"\\n{outputs}\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:17:12.76096Z","iopub.execute_input":"2024-04-30T15:17:12.761807Z","iopub.status.idle":"2024-04-30T15:17:17.73686Z","shell.execute_reply.started":"2024-04-30T15:17:12.761764Z","shell.execute_reply":"2024-04-30T15:17:17.736071Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:17:17.738391Z","iopub.execute_input":"2024-04-30T15:17:17.738683Z","iopub.status.idle":"2024-04-30T15:17:17.744426Z","shell.execute_reply.started":"2024-04-30T15:17:17.738659Z","shell.execute_reply":"2024-04-30T15:17:17.743344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference BBox for image","metadata":{}},{"cell_type":"code","source":"qs = \"Please generate the bounding box for the flower. Bounding box output format is coco. Please think step by step.\"\n \nconv_mode = \"phi3_instruct\"\nimage_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN\nif IMAGE_PLACEHOLDER in qs:\n    if model.config.mm_use_im_start_end:\n        qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs)\n    else:\n        qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs)\nelse:\n    if model.config.mm_use_im_start_end:\n        qs = image_token_se + \"\\n\" + qs\n    else:\n        qs = DEFAULT_IMAGE_TOKEN + \"\\n\" + qs\n\nconv = conv_templates[conv_mode].copy()\nconv.system = \"\"\"<|system|>\\nYou are a helpful visual AI assistant.\"\"\"\nconv.append_message(conv.roles[0], qs)\nconv.append_message(conv.roles[1], None)\nprompt = conv.get_prompt()\n\nprint(prompt)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:32:57.806375Z","iopub.execute_input":"2024-04-30T15:32:57.806752Z","iopub.status.idle":"2024-04-30T15:32:57.814384Z","shell.execute_reply.started":"2024-04-30T15:32:57.806722Z","shell.execute_reply":"2024-04-30T15:32:57.813447Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"saple_img_path = '/kaggle/input/planttraits2024/train_images/100282707.jpeg'\n\nimg = mpimg.imread(saple_img_path)\nplt.imshow(img)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:32:58.653933Z","iopub.execute_input":"2024-04-30T15:32:58.654289Z","iopub.status.idle":"2024-04-30T15:32:58.834539Z","shell.execute_reply.started":"2024-04-30T15:32:58.654263Z","shell.execute_reply":"2024-04-30T15:32:58.833607Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_name = saple_img_path\nimage_files = [image_name]\nimages = load_images(image_files)\nimage_sizes = [x.size for x in images]\nimages_tensor = process_images(\n    images,\n    image_processor,\n    model.config\n).to(model.device, dtype=torch.float16)\n\ninput_ids = (\n    tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\")\n    .unsqueeze(0)\n    .cuda()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:32:59.40369Z","iopub.execute_input":"2024-04-30T15:32:59.404221Z","iopub.status.idle":"2024-04-30T15:32:59.424189Z","shell.execute_reply.started":"2024-04-30T15:32:59.404183Z","shell.execute_reply":"2024-04-30T15:32:59.423364Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temperature = 0.2\ntop_p = 0.7\nnum_beams = 1\nmax_new_tokens = 512\n\nwith torch.inference_mode():\n    output_ids = model.generate(\n        input_ids,\n        images=images_tensor,\n        image_sizes=image_sizes,\n        do_sample=True if temperature > 0 else False,\n        temperature=temperature,\n        top_p=top_p,\n        num_beams=num_beams,\n        max_new_tokens=max_new_tokens,\n        use_cache=True,\n    )\n\noutputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()\noutputs = outputs.replace(\"<|end|>\", \"\").strip()\n#print(f\"\\n{outputs}\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:32:59.802503Z","iopub.execute_input":"2024-04-30T15:32:59.802866Z","iopub.status.idle":"2024-04-30T15:33:01.540892Z","shell.execute_reply.started":"2024-04-30T15:32:59.802833Z","shell.execute_reply":"2024-04-30T15:33:01.539885Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:33:01.542373Z","iopub.execute_input":"2024-04-30T15:33:01.542656Z","iopub.status.idle":"2024-04-30T15:33:01.548537Z","shell.execute_reply.started":"2024-04-30T15:33:01.542632Z","shell.execute_reply":"2024-04-30T15:33:01.547603Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Draw bbox in image","metadata":{}},{"cell_type":"code","source":"import cv2\n\ndef show_box(bbox, saple_img_path, color='red'):\n    image = cv2.imread(f'{saple_img_path}')\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    xmin,ymin,width,height= bbox.split(\",\")\n\n    xmin = int(float(xmin) * image.shape[1]) \n    ymin= int(float(ymin) * image.shape[0])  \n    width = int(float(width) * image.shape[1]) \n    height= int(float(height) * image.shape[0]) \n\n    xmax = xmin + width\n    ymax = ymin + height\n\n    color_tuple = (255,0,0)\n    if color == 'blue':\n        color_tuple = (0,0,255)\n\n    image = cv2.rectangle(image,(xmin,ymin), (xmax,ymax),color_tuple,3)\n\n    img = Image.fromarray(image)\n    return img","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:34:29.484873Z","iopub.execute_input":"2024-04-30T15:34:29.485727Z","iopub.status.idle":"2024-04-30T15:34:29.493181Z","shell.execute_reply.started":"2024-04-30T15:34:29.485691Z","shell.execute_reply":"2024-04-30T15:34:29.492313Z"},"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs = outputs.strip(\"[\")\noutputs = outputs.strip(\"]\")\noutputs","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:38:47.541997Z","iopub.execute_input":"2024-04-30T15:38:47.542425Z","iopub.status.idle":"2024-04-30T15:38:47.549339Z","shell.execute_reply.started":"2024-04-30T15:38:47.542398Z","shell.execute_reply":"2024-04-30T15:38:47.548212Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"_kg_hide-input":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_box(outputs, saple_img_path)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T15:38:51.742517Z","iopub.execute_input":"2024-04-30T15:38:51.743439Z","iopub.status.idle":"2024-04-30T15:38:51.880034Z","shell.execute_reply.started":"2024-04-30T15:38:51.743396Z","shell.execute_reply":"2024-04-30T15:38:51.879137Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- model weight is here: https://www.kaggle.com/models/piantic/llava-phi-3-mini-4k-instruct","metadata":{}}]}