{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8108072,"sourceType":"datasetVersion","datasetId":4789213}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import timm\nimport time\nimport torch\n\nmodel_1 = timm.create_model(\"vit_small_patch14_reg4_dinov2.lvd142m\", dynamic_img_pad=True, dynamic_img_size=True)\nmodel_1.eval()\nmodel_2 = timm.create_model(\"tf_efficientnet_b0.ns_jft_in1k\")\nmodel_2.eval()\nmodel_3 = timm.create_model(\"convnextv2_tiny.fcmae_ft_in22k_in1k_384\")\nmodel_3.eval()\nmodel_4 = timm.create_model(\"convnext_tiny.fb_in22k_ft_in1k_384\")\nmodel_4.eval()\nmodel_5 = timm.create_model(\"efficientvit_b0.r224_in1k\")\nmodel_5.eval()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-22T13:33:39.885535Z","iopub.execute_input":"2024-04-22T13:33:39.886458Z","iopub.status.idle":"2024-04-22T13:33:50.549661Z","shell.execute_reply.started":"2024-04-22T13:33:39.886423Z","shell.execute_reply":"2024-04-22T13:33:50.548859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!lscpu |grep 'Model name'\n!lscpu | grep 'Socket(s):'\n!lscpu | grep 'Core(s) per socket'\n!lscpu | grep 'Thread(s) per core'","metadata":{"execution":{"iopub.status.busy":"2024-04-21T09:50:20.404623Z","iopub.execute_input":"2024-04-21T09:50:20.404933Z","iopub.status.idle":"2024-04-21T09:50:21.614816Z","shell.execute_reply.started":"2024-04-21T09:50:20.404908Z","shell.execute_reply":"2024-04-21T09:50:21.614030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand((1, 3, 518, 518)).float()\nwith torch.no_grad():\n    for i in range(20):\n        t1 = time.time()\n        model_1(x)\n        t2 = time.time()\n        print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:50.551602Z","iopub.execute_input":"2024-04-22T13:33:50.552238Z","iopub.status.idle":"2024-04-22T13:34:10.731169Z","shell.execute_reply.started":"2024-04-22T13:33:50.552209Z","shell.execute_reply":"2024-04-22T13:34:10.729919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for i in range(20):\n        t1 = time.time()\n        model_2(x)\n        t2 = time.time()\n        print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:36:52.115612Z","iopub.execute_input":"2024-04-22T13:36:52.116260Z","iopub.status.idle":"2024-04-22T13:36:55.660190Z","shell.execute_reply.started":"2024-04-22T13:36:52.116227Z","shell.execute_reply":"2024-04-22T13:36:55.659009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for i in range(20):\n        t1 = time.time()\n        model_3(x)\n        t2 = time.time()\n        print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T05:10:48.361292Z","iopub.execute_input":"2024-04-21T05:10:48.361693Z","iopub.status.idle":"2024-04-21T05:11:04.146268Z","shell.execute_reply.started":"2024-04-21T05:10:48.361664Z","shell.execute_reply":"2024-04-21T05:11:04.145117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for i in range(20):\n        t1 = time.time()\n        model_4(x)\n        t2 = time.time()\n        print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:36:31.477467Z","iopub.execute_input":"2024-04-22T13:36:31.477843Z","iopub.status.idle":"2024-04-22T13:36:42.977795Z","shell.execute_reply.started":"2024-04-22T13:36:31.477810Z","shell.execute_reply":"2024-04-22T13:36:42.976679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for i in range(20):\n        t1 = time.time()\n        model_5(x)\n        t2 = time.time()\n        print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:36:26.292099Z","iopub.execute_input":"2024-04-22T13:36:26.292534Z","iopub.status.idle":"2024-04-22T13:36:27.647420Z","shell.execute_reply.started":"2024-04-22T13:36:26.292503Z","shell.execute_reply":"2024-04-22T13:36:27.646344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    t1 = time.time()\n    for i in range(200):\n        model_2(x)\n    t2 = time.time()\n    print(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T09:39:27.396477Z","iopub.execute_input":"2024-04-21T09:39:27.396957Z","iopub.status.idle":"2024-04-21T09:40:01.811712Z","shell.execute_reply.started":"2024-04-21T09:39:27.396927Z","shell.execute_reply":"2024-04-21T09:40:01.810421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.set_num_threads(1)\ndef predict_worker():\n    t1 = time.time()\n    model = timm.create_model(\"tf_efficientnet_b0.ns_jft_in1k\")\n    model.eval()\n    x = torch.rand((1, 3, 518, 518)).float()\n    t2 = time.time()\n    print(\"Model init time: \", t2 - t1)\n    with torch.no_grad():\n        for i in range(50):\n            model(x)\n    t3 = time.time()\n    print(\"Inference time: \", t3 - t2)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-21T09:59:20.052318Z","iopub.execute_input":"2024-04-21T09:59:20.052860Z","iopub.status.idle":"2024-04-21T09:59:20.058877Z","shell.execute_reply.started":"2024-04-21T09:59:20.052832Z","shell.execute_reply":"2024-04-21T09:59:20.057714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport multiprocessing as mp\nfrom joblib import Parallel, delayed\n\nfrom multiprocessing import Process, Queue, Manager, Lock\n\nread_0_process = Process(target=predict_worker, args=())\nread_1_process = Process(target=predict_worker, args=())\nread_2_process = Process(target=predict_worker, args=())\nread_3_process = Process(target=predict_worker, args=())\n\nt1 = time.time()\nread_0_process.start()\nread_1_process.start()\nread_2_process.start()\nread_3_process.start()\n\nread_0_process.join()\nread_1_process.join()\nread_2_process.join()\nread_3_process.join()\nt2 = time.time()\nprint(\"Total inference time: \", t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:17:08.606436Z","iopub.execute_input":"2024-04-21T10:17:08.606756Z","iopub.status.idle":"2024-04-21T10:17:24.936628Z","shell.execute_reply.started":"2024-04-21T10:17:08.606732Z","shell.execute_reply":"2024-04-21T10:17:24.935516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/onnxruntime/humanfriendly-10.0-py2.py3-none-any.whl --no-index --find-links /kaggle/input/onnxruntime\n!pip install /kaggle/input/onnxruntime/coloredlogs-15.0.1-py2.py3-none-any.whl --no-index --find-links /kaggle/input/onnxruntime\n!pip install /kaggle/input/onnxruntime/onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl --no-index --find-links /kaggle/input/onnxruntime","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:02:35.846768Z","iopub.execute_input":"2024-04-21T10:02:35.847102Z","iopub.status.idle":"2024-04-21T10:03:05.715646Z","shell.execute_reply.started":"2024-04-21T10:02:35.847076Z","shell.execute_reply":"2024-04-21T10:03:05.714393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_tensor = torch.randn(1, 3, 518, 518)  # input shape\ninput_names = ['x']\noutput_names = ['output']","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:05:21.515701Z","iopub.execute_input":"2024-04-21T10:05:21.516056Z","iopub.status.idle":"2024-04-21T10:05:21.525151Z","shell.execute_reply.started":"2024-04-21T10:05:21.516030Z","shell.execute_reply":"2024-04-21T10:05:21.524072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntorch.onnx.export(model_2, input_tensor, f\"hehe.onnx\", verbose=False, input_names=input_names, output_names=output_names)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:05:23.345751Z","iopub.execute_input":"2024-04-21T10:05:23.346055Z","iopub.status.idle":"2024-04-21T10:05:25.121061Z","shell.execute_reply.started":"2024-04-21T10:05:23.346032Z","shell.execute_reply":"2024-04-21T10:05:25.120359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnx\nimport onnxruntime as ort\nimport numpy as np\n\nonnx_model = onnx.load(\"hehe.onnx\")\nonnx_model_graph = onnx_model.graph\nonnx_session = ort.InferenceSession(onnx_model.SerializeToString())\n# onnx_session.intra_op_num_threads = 0\n# onnx_session.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL\nonnx_session.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\n\nx = np.zeros((1, 3, 518, 518)).astype(np.float32)\nt1 = time.time()\nfor i in range(200):\n     onnx_session.run(output_names, {input_names[0]: x})\nt2 = time.time()\nprint(t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:16:41.673376Z","iopub.execute_input":"2024-04-21T10:16:41.673718Z","iopub.status.idle":"2024-04-21T10:16:53.360602Z","shell.execute_reply.started":"2024-04-21T10:16:41.673687Z","shell.execute_reply":"2024-04-21T10:16:53.360015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_worker(rank):\n    t1 = time.time()\n    onnx_model = onnx.load(\"hehe.onnx\")\n    onnx_model_graph = onnx_model.graph\n    sess_opt = ort.SessionOptions()\n#     sess_opt.execution_mode  = ort.ExecutionMode.ORT_PARALLEL\n#     sess_opt.intra_op_num_threads = 3\n#     sess_opt.inter_op_num_threads = 2\n#     if rank == 0:\n#         sess_opt.add_session_config_entry('session.intra_op_thread_affinities', '1;2')\n#     else:\n#         sess_opt.add_session_config_entry('session.intra_op_thread_affinities', '3;4')\n    onnx_session = ort.InferenceSession(onnx_model.SerializeToString(), sess_opt)\n    \n\n    x = np.zeros((1, 3, 518, 518)).astype(np.float32)\n    t2 = time.time()\n    print(\"Model init time: \", t2 - t1)\n    for i in range(100):\n        onnx_session.run(output_names, {input_names[0]: x})\n    t3 = time.time()\n    print(\"Inference time: \", t3 - t2)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:26:13.930737Z","iopub.execute_input":"2024-04-21T10:26:13.931070Z","iopub.status.idle":"2024-04-21T10:26:13.937875Z","shell.execute_reply.started":"2024-04-21T10:26:13.931043Z","shell.execute_reply":"2024-04-21T10:26:13.937046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport multiprocessing as mp\nfrom joblib import Parallel, delayed\n\nfrom multiprocessing import Process, Queue, Manager, Lock\n\nread_0_process = Process(target=predict_worker, args=(0,))\nread_1_process = Process(target=predict_worker, args=(1,))\n\nt1 = time.time()\nread_0_process.start()\nread_1_process.start()\n\nread_0_process.join()\nread_1_process.join()\nt2 = time.time()\nprint(\"Total inference time: \", t2 - t1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T10:26:28.832420Z","iopub.execute_input":"2024-04-21T10:26:28.832797Z","iopub.status.idle":"2024-04-21T10:26:39.193899Z","shell.execute_reply.started":"2024-04-21T10:26:28.832765Z","shell.execute_reply":"2024-04-21T10:26:39.193005Z"},"trusted":true},"execution_count":null,"outputs":[]}]}