{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I wrote explanation on [discussions](https://www.kaggle.com/competitions/foursquare-location-matching/discussion/336462\n).\n\nver1: CPU  \nver3: GPU  ","metadata":{}},{"cell_type":"code","source":"from contextlib import contextmanager\nimport math\nimport os\nimport subprocess\nimport sys\nimport time\n\nimport numpy as np\nimport psutil\nimport torch\n\n\ndef get_gpu_memory(cmd_path=\"nvidia-smi\",\n                   target_properties=(\"memory.total\", \"memory.used\")):\n    \"\"\"\n    ref: https://www.12-technology.com/2022/01/pythongpu.html\n    Returns\n    -------\n    gpu_total : ndarray,  \"memory.total\"\n    gpu_used: ndarray, \"memory.used\"\n    \"\"\"\n\n    # format option\n    format_option = \"--format=csv,noheader,nounits\"\n\n    cmd = '%s --query-gpu=%s %s' % (cmd_path, ','.join(target_properties), format_option)\n\n    # Command execution in sub-processes\n    cmd_res = subprocess.check_output(cmd, shell=True)\n\n    gpu_lines = cmd_res.decode().split('\\n')[0].split(', ')\n\n    gpu_total = int(gpu_lines[0]) / 1024\n    gpu_used = int(gpu_lines[1]) / 1024\n\n    gpu_total = np.round(gpu_used, 1)\n    gpu_used = np.round(gpu_used, 1)\n    return gpu_total, gpu_used\n\n\nclass Trace():\n    cuda = torch.cuda.is_available()\n\n    @contextmanager\n    def timer(self, title):\n        t0 = time.time()\n        p = psutil.Process(os.getpid())\n        cpu_m0 = p.memory_info().rss / 2. ** 30\n        if self.cuda: gpu_m0 = get_gpu_memory()[0]\n        yield\n        cpu_m1 = p.memory_info().rss / 2. ** 30\n        if self.cuda: gpu_m1 = get_gpu_memory()[0]\n\n        cpu_delta = cpu_m1 - cpu_m0\n        if self.cuda: gpu_delta = gpu_m1 - gpu_m0\n\n        cpu_sign = '+' if cpu_delta >= 0 else '-'\n        cpu_delta = math.fabs(cpu_delta)\n\n        if self.cuda: gpu_sign = '+' if gpu_delta >= 0 else '-'\n        if self.cuda: gpu_delta = math.fabs(gpu_delta)\n\n        cpu_message = f'{cpu_m1:.1f}GB({cpu_sign}{cpu_delta:.1f}GB)'\n        if self.cuda: gpu_message = f'{gpu_m1:.1f}GB({gpu_sign}{gpu_delta:.1f}GB)'\n\n        if self.cuda:\n            message = f\"[cpu: {cpu_message}, gpu: {gpu_message}: {time.time() - t0:.1f}sec] {title} \"\n        else:\n            message = f\"[cpu: {cpu_message}: {time.time() - t0:.1f}sec] {title} \"\n\n        print(message, file=sys.stderr)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T09:53:34.501135Z","iopub.execute_input":"2022-07-11T09:53:34.501646Z","iopub.status.idle":"2022-07-11T09:53:34.522381Z","shell.execute_reply.started":"2022-07-11T09:53:34.501603Z","shell.execute_reply":"2022-07-11T09:53:34.521307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trace = Trace()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T09:53:34.524377Z","iopub.execute_input":"2022-07-11T09:53:34.524831Z","iopub.status.idle":"2022-07-11T09:53:34.537613Z","shell.execute_reply.started":"2022-07-11T09:53:34.524792Z","shell.execute_reply":"2022-07-11T09:53:34.536415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport cudf\nimport gc\n\nwith trace.timer('read train'):\n    data = cudf.read_csv('../input/foursquare-location-matching/train.csv')\nwith trace.timer('del train'):\n    del data\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T09:53:49.632296Z","iopub.execute_input":"2022-07-11T09:53:49.632658Z","iopub.status.idle":"2022-07-11T09:53:59.924178Z","shell.execute_reply.started":"2022-07-11T09:53:49.632627Z","shell.execute_reply":"2022-07-11T09:53:59.922843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}