{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Check hardware","metadata":{}},{"cell_type":"code","source":"import cv2\n\nrun_inference = False\ntry:\n    x = cv2.imread(\"/kaggle/input/vesuvius-challenge-ink-detection/test/a/mask.png\", 0)\n    if x.shape != (2727, 6330):\n        run_inference = True\nexcept:\n    run_inference = True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-11T18:03:59.382066Z","iopub.execute_input":"2023-06-11T18:03:59.382736Z","iopub.status.idle":"2023-06-11T18:04:00.396593Z","shell.execute_reply.started":"2023-06-11T18:03:59.382706Z","shell.execute_reply":"2023-06-11T18:04:00.395393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!lscpu |grep 'Model name'\n#no.of cores each processor is having\n!lscpu | grep 'Core(s) per socket'\n#no.of threads each core is having\n!lscpu | grep 'Thread(s) per core'","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:04:00.399115Z","iopub.execute_input":"2023-06-11T18:04:00.399602Z","iopub.status.idle":"2023-06-11T18:04:04.247389Z","shell.execute_reply.started":"2023-06-11T18:04:00.399564Z","shell.execute_reply":"2023-06-11T18:04:04.246135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install deps","metadata":{}},{"cell_type":"code","source":"if run_inference:\n    !pip \\\n        install \\\n        --no-index \\\n        --find-links /kaggle/input/scrolls-software/software/ \\\n        lightning==2.0.2 \\\n        timm==0.9.2 \\\n        patchify==0.2.3 \\\n        imagesize==1.4.1 \\\n        jsonargparse[signatures]","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:04:04.26059Z","iopub.execute_input":"2023-06-11T18:04:04.266959Z","iopub.status.idle":"2023-06-11T18:04:22.68631Z","shell.execute_reply.started":"2023-06-11T18:04:04.266921Z","shell.execute_reply":"2023-06-11T18:04:22.685047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    !mkdir /kaggle/working/software-aux\n    !cp /kaggle/input/scrolls-software/software/ACSConv-0.1.1.tar.gz.nounzip \\\n        /kaggle/working/software-aux/ACSConv-0.1.1.tar.gz ","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:04:22.688855Z","iopub.execute_input":"2023-06-11T18:04:22.689618Z","iopub.status.idle":"2023-06-11T18:04:24.608019Z","shell.execute_reply.started":"2023-06-11T18:04:22.689587Z","shell.execute_reply":"2023-06-11T18:04:24.606595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    !pip \\\n        install \\\n        --no-index \\\n        --find-links /kaggle/working/software-aux \\\n        ACSConv==0.1.1","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:04:24.610692Z","iopub.execute_input":"2023-06-11T18:04:24.611355Z","iopub.status.idle":"2023-06-11T18:04:38.474963Z","shell.execute_reply.started":"2023-06-11T18:04:24.611315Z","shell.execute_reply":"2023-06-11T18:04:38.473821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    !dpkg -i --force-depends /kaggle/input/pyvips-offline-installer/archives/*.deb >/dev/null 2>&1\n    !pip install --no-index --find-links /kaggle/input/pyvips-offline-installer cffi==1.15.1 \n    !pip install --no-index --find-links /kaggle/input/pyvips-offline-installer pycparser==2.21\n    !pip install --no-index --find-links /kaggle/input/pyvips-offline-installer pyvips==2.2.1","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:04:38.477037Z","iopub.execute_input":"2023-06-11T18:04:38.477422Z","iopub.status.idle":"2023-06-11T18:10:21.817448Z","shell.execute_reply.started":"2023-06-11T18:04:38.477385Z","shell.execute_reply":"2023-06-11T18:10:21.81604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Update env","metadata":{}},{"cell_type":"code","source":"!wandb disabled","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:10:21.820498Z","iopub.execute_input":"2023-06-11T18:10:21.821562Z","iopub.status.idle":"2023-06-11T18:10:24.812217Z","shell.execute_reply.started":"2023-06-11T18:10:21.821514Z","shell.execute_reply":"2023-06-11T18:10:24.811085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['PYTHONPATH'] = \"/kaggle/input/scrolls:\" + os.environ['PYTHONPATH']\nos.environ['LOGLEVEL'] = \"ERROR\"\n!echo $PYTHONPATH","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:10:24.816987Z","iopub.execute_input":"2023-06-11T18:10:24.817334Z","iopub.status.idle":"2023-06-11T18:10:25.779245Z","shell.execute_reply.started":"2023-06-11T18:10:24.817304Z","shell.execute_reply":"2023-06-11T18:10:25.774364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build Z shift & scale maps","metadata":{}},{"cell_type":"code","source":"if run_inference:\n    # Build downscaled version\n    args = ' '.join(\n        [\n            f'/kaggle/input/vesuvius-challenge-ink-detection/test',\n            f'/kaggle/working/fragments_downscaled_2/test',\n            '2',\n        ]\n    )\n    !python /kaggle/input/scrolls/src/scripts/downscale_dataset.py {args}\n\n    # Build shift & scale maps\n    for fragment_id in ['a', 'b']:\n        print(f'Build shift & scale maps for {fragment_id}')\n        args = ' '.join(\n            [\n                f'--input_dir /kaggle/input/vesuvius-challenge-ink-detection/test/{fragment_id}',\n                f'--downscaled_input_dir /kaggle/working/fragments_downscaled_2/test/{fragment_id}',\n                f'--output_dir /kaggle/working/fragments_downscaled_2/test/{fragment_id}',\n                '--patch_size 256',\n                '--downscale_factor 2',\n                '--overlap_divider 2',\n            ]\n        )\n        !python /kaggle/input/scrolls/src/scripts/z_shift_scale/build_maps.py {args}\n\n    # Apply z shift and scale\n    for fragment_id in ['a', 'b']:\n        args = ' '.join(\n            [\n                f'--input_dir /kaggle/input/vesuvius-challenge-ink-detection/test/{fragment_id}',\n                f'--output_dir /kaggle/working/fragments_z_shift_scale/test/{fragment_id}',\n                f'--map_path /kaggle/working/fragments_downscaled_2/test/{fragment_id}',\n                '--z_start 20',\n                '--z_end 44',\n            ]\n        )\n        !python /kaggle/input/scrolls/src/scripts/z_shift_scale/scale_online.py {args}","metadata":{"execution":{"iopub.status.busy":"2023-06-11T18:21:20.303034Z","iopub.execute_input":"2023-06-11T18:21:20.303787Z","iopub.status.idle":"2023-06-11T18:40:56.370374Z","shell.execute_reply.started":"2023-06-11T18:21:20.303748Z","shell.execute_reply":"2023-06-11T18:40:56.369163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### (Optional) validation run without TTA","metadata":{}},{"cell_type":"code","source":"# !python /kaggle/input/scrolls/run/main.py predict \\\n#     --config /kaggle/input/scrolls/run/configs/2d-agg/unet-2d-agg.yaml \\\n#     --config /kaggle/input/scrolls/run/configs/2d/maxvit.yaml \\\n#     --model.init_args.pretrained false \\\n#     --model.init_args.tta_each_n_epochs -1 \\\n#     --model.init_args.tta_n_random_replays 0 \\\n#     --data.init_args.use_online_val_test true \\\n#     --data.init_args.z_start 20 \\\n#     --data.init_args.z_end 44 \\\n#     --data.init_args.num_workers 2 \\\n#     --data.init_args.surface_volume_dirs null \\\n#     --data.init_args.surface_volume_dirs_test \"[/kaggle/working/fragments_z_shift_scale/test/a]\" \\\n#     --data.init_args.val_dir_indices null \\\n#     --ckpt_path /kaggle/input/scrolls-best-maxvit-5-fold-cv/fold_0/checkpoints/epoch=43-step=4620.ckpt","metadata":{"execution":{"iopub.status.busy":"2023-06-11T19:00:06.18309Z","iopub.execute_input":"2023-06-11T19:00:06.184203Z","iopub.status.idle":"2023-06-11T19:04:36.246208Z","shell.execute_reply.started":"2023-06-11T19:00:06.184115Z","shell.execute_reply":"2023-06-11T19:04:36.244959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"markdown","source":"Predict with each fold model and each architecture with configured TTA.","metadata":{}},{"cell_type":"code","source":"import gc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    fold_index_to_ckpt_pathes = {\n        0: '/kaggle/input/scrolls-best-maxvit-agg-no-val-folds-0-3/seed_0/checkpoints/best.ckpt',\n        1: '/kaggle/input/scrolls-best-maxvit-agg-no-val-folds-0-3/seed_1/checkpoints/best.ckpt',\n        2: '/kaggle/input/scrolls-best-maxvit-agg-no-val-folds-0-3/seed_2/checkpoints/best.ckpt',\n        3: '/kaggle/input/scrolls-best-maxvit-agg-no-val-folds-0-3/seed_3/checkpoints/best.ckpt',\n        4: '/kaggle/input/scrolls-best-maxvit-agg-no-val-fold-4/best.ckpt',\n    }\n\n    z_start_start = 20\n    z_start_end = 21\n    tta_each_n_epochs = 1\n\n    for z_start in range(z_start_start, z_start_end):\n        z_end = z_start + 24\n        for fold_index in [0, 1, 2, 3, 4]:    \n            ckpt_path = fold_index_to_ckpt_pathes[fold_index]\n            for fragment_id in ['a', 'b']:\n                gc.collect()\n                args = ' '.join(\n                    [\n                        'predict',\n                        '--config /kaggle/input/scrolls/run/configs/2d-agg/unet-2d-agg.yaml',\n                        '--config /kaggle/input/scrolls/run/configs/2d/maxvit.yaml',\n                        '--model.init_args.pretrained false',\n                        f'--model.init_args.tta_each_n_epochs {tta_each_n_epochs}',\n\n                        # pre-scaled but still online\n                        '--data.init_args.use_online_val_test true',\n                        f'--data.init_args.z_shift_scale_pathes_test null',\n\n                        f'--data.init_args.z_start {z_start}',\n                        f'--data.init_args.z_end {z_end}',\n                        '--data.init_args.num_workers 2',\n                        '--data.init_args.surface_volume_dirs null',\n                        '--data.init_args.val_dir_indices null',\n                        f'--data.init_args.surface_volume_dirs_test \"[/kaggle/working/fragments_z_shift_scale/test/{fragment_id}]\"',\n                        f'--ckpt_path {ckpt_path}',\n                    ]\n                )\n                !python /kaggle/input/scrolls/run/main.py {args}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls *.png","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Average","metadata":{}},{"cell_type":"code","source":"if run_inference:\n    !rm submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:38:41.399493Z","iopub.execute_input":"2023-06-04T14:38:41.399938Z","iopub.status.idle":"2023-06-04T14:38:42.427351Z","shell.execute_reply.started":"2023-06-04T14:38:41.399899Z","shell.execute_reply":"2023-06-04T14:38:42.426045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    !python \\\n        /kaggle/input/scrolls/src/scripts/aggregate_images.py \\\n        . \\\n        .","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:38:43.378955Z","iopub.execute_input":"2023-06-04T14:38:43.379691Z","iopub.status.idle":"2023-06-04T14:38:58.962221Z","shell.execute_reply.started":"2023-06-04T14:38:43.379652Z","shell.execute_reply":"2023-06-04T14:38:58.96104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if run_inference:\n    !rm -r /kaggle/working/fragments_downscaled_2\n    !rm -r /kaggle/working/fragments_z_shift_scale\n    !rm -r /kaggle/working/software-aux\n    !rm -r /kaggle/working/tmp\n    !rm -r /kaggle/working/wandb\n    !rm *.png","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not run_inference:\n    !cp /kaggle/input/vesuvius-challenge-ink-detection/sample_submission.csv submission.csv ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:01:18.248376Z","iopub.execute_input":"2023-06-05T12:01:18.248745Z","iopub.status.idle":"2023-06-05T12:01:19.252322Z","shell.execute_reply.started":"2023-06-05T12:01:18.248718Z","shell.execute_reply":"2023-06-05T12:01:19.251149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}