{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"},{"sourceId":1664210,"sourceType":"datasetVersion","datasetId":985406},{"sourceId":1664236,"sourceType":"datasetVersion","datasetId":985422},{"sourceId":1664248,"sourceType":"datasetVersion","datasetId":985433},{"sourceId":4319351,"sourceType":"datasetVersion","datasetId":2544012}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# original environment: python 3.7, no torch 2 version for this, need python 3.8 ... 3.11\n#!pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu118","metadata":{"execution":{"iopub.status.busy":"2024-07-15T11:03:24.233616Z","iopub.execute_input":"2024-07-15T11:03:24.234073Z","iopub.status.idle":"2024-07-15T11:03:26.831655Z","shell.execute_reply.started":"2024-07-15T11:03:24.234024Z","shell.execute_reply":"2024-07-15T11:03:26.830405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleanup\n%cd /kaggle/working\n!rm -rf deeptrane\n#!rm *\n!touch 0.txt\n\n!git clone -b autolevels https://github.com/yellowdolphin/deeptrane.git\n%cd /kaggle/working/deeptrane\n#!git checkout 5a9132c401cd8d8b09\n\n# PyTorch\nlr = 1e-3  # ignored when restarting with opt state\nn_acc = 1\n\n#!python train.py -c cassava_resnet18_v000 --epochs 10  --size 64 --set batch_verbose 1 \\\n#!python train.py -c cassava_resnet18_v000 --epochs 10  --size 128 --set use_dp False \\\n#--freeze none --set bn_head True  --lin_ftrs 25 --dropout_ps 0.2 # 5.7-7.1 min @ T4x1\n#!python train.py -c cassava_resnet18_v000 --epochs 10  --size 128 --set use_dp True  # 3.4-7.2 min @ T4x2 DP\n#!torchrun --standalone --nnodes=1 --nproc_per_node=2 \\\n#        train.py -c cassava_resnet18_v000 --epochs 10  --size 128  # 6.2-6.5 min @ T4x2 DDP\n\n!python train.py -c cassava_resnet18_v000 --epochs 5 --size 224 --set bs 32 \\\n--set arch_name efficientnet_lite0.ra_in1k \\\n--set lr_head {lr} --set lr_bn {lr} --set lr_body {lr} --set one_cycle True --set n_acc {n_acc} \\\n--set bn_momentum 0.1 --set bn_eps 1e-5 \\\n--set num_workers 4 --set dtype float32 \\\n#--set DEBUG True\n\n# Tensorflow\n#!python train_tf.py -c cassava_resnet18_tf --epochs 10  --set batch_verbose 1  --size 128  # 1.6 min @ TPUx8\n#!python train_tf.py -c cassava_resnet18_tf --epochs 10  --set batch_verbose 1  --size 128  # 24.1 min @ T4x1\n#!python train_tf.py -c cassava_resnet18_tf --epochs 10  --set batch_verbose 1  --size 64 \\\n#--set arch_name deit_small_patch16_224 --set frac 0.001\n#--set arch_name efnv2s \\ # ok","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-15T13:07:17.910937Z","iopub.execute_input":"2024-07-15T13:07:17.911335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot metrics","metadata":{}},{"cell_type":"code","source":"# Cleanup\n#!rm -r deeptrane\n%cd /kaggle/working\n\n# Plot Results (PyTorch and TF)\nfrom deeptrane.utils.plot import plot_metrics\nplot_metrics()","metadata":{"execution":{"iopub.status.busy":"2024-07-14T21:51:08.156557Z","iopub.execute_input":"2024-07-14T21:51:08.156880Z","iopub.status.idle":"2024-07-14T21:51:11.382745Z","shell.execute_reply.started":"2024-07-14T21:51:08.156843Z","shell.execute_reply":"2024-07-14T21:51:11.381551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat metrics_fold0.csv","metadata":{"execution":{"iopub.status.busy":"2024-07-14T21:51:11.385636Z","iopub.execute_input":"2024-07-14T21:51:11.386498Z","iopub.status.idle":"2024-07-14T21:51:12.461371Z","shell.execute_reply.started":"2024-07-14T21:51:11.386435Z","shell.execute_reply":"2024-07-14T21:51:12.459945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Debugging","metadata":{}},{"cell_type":"code","source":"import os, platform, subprocess\n\ndef get_processor_name():\n    if platform.system() == \"Windows\":\n        return platform.processor()\n    elif platform.system() == \"Darwin\":\n        os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/sbin'\n        command =\"sysctl -n machdep.cpu.brand_string\"\n        return subprocess.check_output(command).strip()\n    elif platform.system() == \"Linux\":\n        command = \"grep 'model name' /proc/cpuinfo\"\n        all_info = subprocess.check_output(command, shell=True).decode().strip()\n        return all_info.split(\"\\n\")[0].split(': ')[1]\n    return \"\"\n\nprint(get_processor_name())\n\nfor k, v in os.environ.items():\n    print(f\"{k:<30}  {v[:120]}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-15T11:01:22.756890Z","iopub.execute_input":"2024-07-15T11:01:22.757240Z","iopub.status.idle":"2024-07-15T11:01:22.796178Z","shell.execute_reply.started":"2024-07-15T11:01:22.757205Z","shell.execute_reply":"2024-07-15T11:01:22.794886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DEBUG Exception in `AtomicFunction.__del__`\n\n```pytthon\nException ignored in: <function AtomicFunction.__del__ at 0x7b021d90fd90>\nTraceback (most recent call last):\n  File \"/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py\", line 218, in __del__\nTypeError: 'NoneType' object is not subscriptable\n```\n\nThis warning is raised since TF v2.13.0, but not in 2.12.1.\n\n### Solution\n`del dataset` after training to avoid this garbage collection warning.","metadata":{}},{"cell_type":"markdown","source":"# Why so slow on CPU?\n\nefn_lite0 runs at 53 min/ep (19.1 min/ep 1y ago, see v140 or 18.9 min/ep v372), due to \n- num_workers=4 instead 8? no\n- BN momentum, eps? no, but has minor effect\n- code changes? no\n    - same deeptrane commit as in v140 also takes 52 min/ep\n- latest env? maybe: env matters\n    - v372 ran on KAGGLE_CONTAINER_NAME=kaggle_MEplExLx8qJnZdHrsByqE6KWplR8d8iWy0JUpkomA-136805707-webtier\n    - env 2023-07-10: same BUILD_DATE, KAGGLE_CONTAINER_NAME differs, region/hardware may differ, 2023-06-28 commit takes 29.9 min/ep\n- dataset access speed/bucket location? no\n    - cp is fast, train on local files takes same time\n- GPU runtime affected? no, GPU run got even faster\n    - 1y ago evn_lite0 took 1.4 min/ep (1.6 min/ep with AMP/float16)\n    - current code in latest env: 1.1 min/ep (v387)","metadata":{}},{"cell_type":"code","source":"# TF minimal test code\nscript = \"\"\"\nfrom multiprocessing import cpu_count\nimport tensorflow as tf\nprint(\"TF:\", tf.__version__)\n\nraw_dataset = tf.data.TFRecordDataset(['example.tfrec'])\n#del raw_dataset\n\"\"\"\nif False:\n    with open('test.py', 'w') as fp:\n        fp.write(script)\n\n    !python test.py","metadata":{"execution":{"iopub.status.busy":"2024-07-15T11:01:10.259374Z","iopub.execute_input":"2024-07-15T11:01:10.259782Z","iopub.status.idle":"2024-07-15T11:01:10.269705Z","shell.execute_reply.started":"2024-07-15T11:01:10.259745Z","shell.execute_reply":"2024-07-15T11:01:10.268197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}