{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is the code to create the dataset for training/validation","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-01T06:42:19.913260Z","iopub.execute_input":"2023-06-01T06:42:19.913685Z","iopub.status.idle":"2023-06-01T06:42:26.089112Z","shell.execute_reply.started":"2023-06-01T06:42:19.913653Z","shell.execute_reply":"2023-06-01T06:42:26.088022Z"}}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score, log_loss\nimport pickle\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport warnings\nimport sys\nimport pandas as pd\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport cv2\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport argparse\nimport importlib\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD, AdamW\n\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:25.022246Z","iopub.execute_input":"2023-06-01T07:59:25.022743Z","iopub.status.idle":"2023-06-01T07:59:34.932173Z","shell.execute_reply.started":"2023-06-01T07:59:25.022702Z","shell.execute_reply":"2023-06-01T07:59:34.930448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:34.934427Z","iopub.execute_input":"2023-06-01T07:59:34.935335Z","iopub.status.idle":"2023-06-01T07:59:37.406395Z","shell.execute_reply.started":"2023-06-01T07:59:34.935277Z","shell.execute_reply":"2023-06-01T07:59:37.404640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"markdown","source":"**We can change the stride, tile size, number of channels etc here**","metadata":{}},{"cell_type":"code","source":"import os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass CFG:\n    # ============== comp exp name =============\n    comp_name = 'vesuvius'\n\n    # comp_dir_path = './'\n    comp_dir_path = '/kaggle/input/'\n    comp_folder_name = 'vesuvius-challenge-ink-detection'\n    # comp_dataset_path = f'{comp_dir_path}datasets/{comp_folder_name}/'\n    comp_dataset_path = f'{comp_dir_path}{comp_folder_name}/'\n    \n    exp_name = 'vesuvius_2d_slide_exp006'\n\n\n    target_size = 1\n    in_chans = 3 # 65\n    seed = 42\n    # ============== training cfg =============\n    size = 224\n    tile_size = 224\n    stride = tile_size\n\n    # ============== set dataset path =============\n    print('set dataset path')\n\n    outputs_path = f'/kaggle/working/outputs/{comp_name}/{exp_name}/'\n\n    data_dir = outputs_path + \\\n        f'{comp_name}-data/'\n\n    log_dir = outputs_path + 'logs/'\n    log_path = log_dir + f'{exp_name}.txt'\n\n    # ============== augmentation =============\n    train_aug_list = [\n        # A.RandomResizedCrop(\n        #     size, size, scale=(0.85, 1.0)),\n        A.Resize(size, size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomBrightnessContrast(p=0.75),\n        A.ShiftScaleRotate(p=0.75),\n        A.OneOf([\n                A.GaussNoise(var_limit=[10, 50]),\n                A.GaussianBlur(),\n                A.MotionBlur(),\n                ], p=0.4),\n        A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.5),\n        A.CoarseDropout(max_holes=1, max_width=int(size * 0.3), max_height=int(size * 0.3), \n                        mask_fill_value=0, p=0.5),\n        # A.Cutout(max_h_size=int(size * 0.6),\n        #          max_w_size=int(size * 0.6), num_holes=1, p=1.0),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n\n    valid_aug_list = [\n        A.Resize(size, size),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.408421Z","iopub.execute_input":"2023-06-01T07:59:37.409176Z","iopub.status.idle":"2023-06-01T07:59:37.429186Z","shell.execute_reply.started":"2023-06-01T07:59:37.409102Z","shell.execute_reply":"2023-06-01T07:59:37.426533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_logger(log_file):\n    from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\ndef set_seed(seed=None, cudnn_deterministic=True):\n    if seed is None:\n        seed = 42\n\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = cudnn_deterministic\n    torch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.432845Z","iopub.execute_input":"2023-06-01T07:59:37.433729Z","iopub.status.idle":"2023-06-01T07:59:37.470245Z","shell.execute_reply.started":"2023-06-01T07:59:37.433663Z","shell.execute_reply":"2023-06-01T07:59:37.469086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_dirs(cfg):\n    for dir in [cfg.data_dir,cfg.log_dir]:\n        os.makedirs(dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.471647Z","iopub.execute_input":"2023-06-01T07:59:37.472362Z","iopub.status.idle":"2023-06-01T07:59:37.496927Z","shell.execute_reply.started":"2023-06-01T07:59:37.472325Z","shell.execute_reply":"2023-06-01T07:59:37.495625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cfg_init(cfg, mode='train'):\n    set_seed(cfg.seed)\n    # set_env_name()\n    # set_dataset_path(cfg)\n\n    if mode == 'train':\n        make_dirs(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.499637Z","iopub.execute_input":"2023-06-01T07:59:37.500529Z","iopub.status.idle":"2023-06-01T07:59:37.529015Z","shell.execute_reply.started":"2023-06-01T07:59:37.500493Z","shell.execute_reply":"2023-06-01T07:59:37.527277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg_init(CFG)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nLogger = init_logger(log_file=CFG.log_path)\n\nLogger.info('\\n\\n-------- exp_info -----------------')\n# Logger.info(datetime.datetime.now().strftime('%Y年%m月%d日 %H:%M:%S'))","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.531414Z","iopub.execute_input":"2023-06-01T07:59:37.532212Z","iopub.status.idle":"2023-06-01T07:59:37.570827Z","shell.execute_reply.started":"2023-06-01T07:59:37.532146Z","shell.execute_reply":"2023-06-01T07:59:37.568557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Change the input channels according to trainig config","metadata":{}},{"cell_type":"code","source":"def read_image_mask(fragment_id):\n\n    images = []\n\n    # idxs = range(65)\n    mid = 65 // 2\n    start = 29\n    end = 32\n#     start = mid - CFG.in_chans // 2\n#     end = mid + CFG.in_chans // 2\n    idxs = range(start, end)\n\n    for i in tqdm(idxs):\n        \n        image = cv2.imread(CFG.comp_dataset_path + f\"train/{fragment_id}/surface_volume/{i:02}.tif\", 0)\n\n        pad0 = (CFG.tile_size - image.shape[0] % CFG.tile_size)\n        pad1 = (CFG.tile_size - image.shape[1] % CFG.tile_size)\n\n        image = np.pad(image, [(0, pad0), (0, pad1)], constant_values=0)\n\n        images.append(image)\n    images = np.stack(images, axis=2)\n\n    mask = cv2.imread(CFG.comp_dataset_path + f\"train/{fragment_id}/inklabels.png\", 0)\n    mask = np.pad(mask, [(0, pad0), (0, pad1)], constant_values=0)\n\n    mask = mask.astype('float32')\n    mask /= 255.0\n    \n    return images, mask","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:37.572474Z","iopub.execute_input":"2023-06-01T07:59:37.572913Z","iopub.status.idle":"2023-06-01T07:59:37.588415Z","shell.execute_reply.started":"2023-06-01T07:59:37.572878Z","shell.execute_reply":"2023-06-01T07:59:37.586916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_valid_dataset():\n    images = []\n    masks = []\n    xyxys = []\n    fragment_no = []\n\n    for fragment_id in range(1, 4):\n\n        image, mask = read_image_mask(fragment_id)\n        print(image.shape)\n\n        x1_list = list(range(0, image.shape[1]-CFG.tile_size+1, CFG.stride))\n        y1_list = list(range(0, image.shape[0]-CFG.tile_size+1, CFG.stride))\n\n        for y1 in y1_list:\n            for x1 in x1_list:\n                y2 = y1 + CFG.tile_size\n                x2 = x1 + CFG.tile_size\n                # xyxys.append((x1, y1, x2, y2))\n        \n                \n                images.append(image[y1:y2, x1:x2])\n                masks.append(mask[y1:y2, x1:x2, None])\n                fragment_no.append(fragment_id)\n                xyxys.append([x1, y1, x2, y2])\n               \n\n    return images, masks, xyxys, fragment_no","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:01:27.228592Z","iopub.execute_input":"2023-06-01T08:01:27.229660Z","iopub.status.idle":"2023-06-01T08:01:27.239491Z","shell.execute_reply.started":"2023-06-01T08:01:27.229620Z","shell.execute_reply":"2023-06-01T08:01:27.238282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, masks, xyxys, frag_no = get_train_valid_dataset()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:01:29.002021Z","iopub.execute_input":"2023-06-01T08:01:29.002491Z","iopub.status.idle":"2023-06-01T08:01:41.276622Z","shell.execute_reply.started":"2023-06-01T08:01:29.002457Z","shell.execute_reply":"2023-06-01T08:01:41.275088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xyxys = np.stack(xyxys)   USE THIS CODE FOR TRAINING LATER ON ","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:02:14.164125Z","iopub.execute_input":"2023-06-01T08:02:14.165373Z","iopub.status.idle":"2023-06-01T08:02:14.179507Z","shell.execute_reply.started":"2023-06-01T08:02:14.165268Z","shell.execute_reply":"2023-06-01T08:02:14.176098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(images),len(masks),len(xyxys),len(frag_no)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:02:25.586279Z","iopub.execute_input":"2023-06-01T08:02:25.586729Z","iopub.status.idle":"2023-06-01T08:02:25.599266Z","shell.execute_reply.started":"2023-06-01T08:02:25.586671Z","shell.execute_reply":"2023-06-01T08:02:25.596786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(images),type(masks)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:03:31.860047Z","iopub.execute_input":"2023-06-01T08:03:31.860463Z","iopub.status.idle":"2023-06-01T08:03:31.868898Z","shell.execute_reply.started":"2023-06-01T08:03:31.860429Z","shell.execute_reply":"2023-06-01T08:03:31.867371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'Frag_no': pd.Series(frag_no),\n                   'Images': pd.Series(images),\n                   'Masks': pd.Series(masks),\n                   'XYXYS': pd.Series(xyxys)})","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:03:33.885839Z","iopub.execute_input":"2023-06-01T08:03:33.886241Z","iopub.status.idle":"2023-06-01T08:03:33.938431Z","shell.execute_reply.started":"2023-06-01T08:03:33.886210Z","shell.execute_reply":"2023-06-01T08:03:33.937189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T08:03:36.949185Z","iopub.execute_input":"2023-06-01T08:03:36.949911Z","iopub.status.idle":"2023-06-01T08:04:24.740837Z","shell.execute_reply.started":"2023-06-01T08:03:36.949860Z","shell.execute_reply":"2023-06-01T08:04:24.739556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:44.847867Z","iopub.status.idle":"2023-06-01T07:59:44.848534Z","shell.execute_reply.started":"2023-06-01T07:59:44.848213Z","shell.execute_reply":"2023-06-01T07:59:44.848243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_csv = CFG.data_dir+'data.csv'\ndf.to_csv(data_csv, index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T07:59:44.852576Z","iopub.status.idle":"2023-06-01T07:59:44.853618Z","shell.execute_reply.started":"2023-06-01T07:59:44.853341Z","shell.execute_reply":"2023-06-01T07:59:44.853378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}