{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport pydicom as pydi\nimport numpy as np\n\nimport math\nimport time\nimport tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport matplotlib.patches as patches\n\nimport glob\nimport random\nimport sys\nfrom PIL import Image\nfrom torch.utils.data import Dataset\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nimport cv2\nimport pydicom as pydi","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-11T20:44:13.4621Z","iopub.execute_input":"2021-11-11T20:44:13.462467Z","iopub.status.idle":"2021-11-11T20:44:15.078924Z","shell.execute_reply.started":"2021-11-11T20:44:13.462378Z","shell.execute_reply":"2021-11-11T20:44:15.077844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clases y funciones UTILS YOLO","metadata":{}},{"cell_type":"code","source":"def to_cpu(tensor):\n    return tensor.detach().cpu()\n\n\ndef load_classes(path):\n    fp = open(path, \"r\")\n    names = fp.read().split(\"\\n\")[:-1]\n    return names\n\n\ndef weights_init_normal(m):\n    classname = m.__class__.__name__\n    if classname.find(\"Conv\") != -1:\n        torch.nn.init.normal_(m.weight.data, 0.0, 0.02)\n    elif classname.find(\"BatchNorm2d\") != -1:\n        torch.nn.init.normal_(m.weight.data, 1.0, 0.02)\n        torch.nn.init.constant_(m.bias.data, 0.0)\n\n\ndef rescale_boxes(boxes, current_dim, original_shape):\n    orig_h, orig_w = original_shape\n    pad_x = max(orig_h - orig_w, 0) * (current_dim / max(original_shape))\n    pad_y = max(orig_w - orig_h, 0) * (current_dim / max(original_shape))\n    unpad_h = current_dim - pad_y\n    unpad_w = current_dim - pad_x\n    boxes[:, 0] = ((boxes[:, 0] - pad_x // 2) / unpad_w) * orig_w\n    boxes[:, 1] = ((boxes[:, 1] - pad_y // 2) / unpad_h) * orig_h\n    boxes[:, 2] = ((boxes[:, 2] - pad_x // 2) / unpad_w) * orig_w\n    boxes[:, 3] = ((boxes[:, 3] - pad_y // 2) / unpad_h) * orig_h\n    return boxes\n\n\ndef xywh2xyxy(x):\n    y = x.new(x.shape)\n    y[..., 0] = x[..., 0] - x[..., 2] / 2\n    y[..., 1] = x[..., 1] - x[..., 3] / 2\n    y[..., 2] = x[..., 0] + x[..., 2] / 2\n    y[..., 3] = x[..., 1] + x[..., 3] / 2\n    return y\n\n\ndef ap_per_class(tp, conf, pred_cls, target_cls):\n    i = np.argsort(-conf)\n    tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n    unique_classes = np.unique(target_cls)\n    ap, p, r = [], [], []\n    for c in tqdm.tqdm(unique_classes, desc=\"Computing AP\"):\n        i = pred_cls == c\n        n_gt = (target_cls == c).sum()  \n        n_p = i.sum()\n\n        if n_p == 0 and n_gt == 0:\n            continue\n        elif n_p == 0 or n_gt == 0:\n            ap.append(0)\n            r.append(0)\n            p.append(0)\n        else:\n            fpc = (1 - tp[i]).cumsum()\n            tpc = (tp[i]).cumsum()\n\n            recall_curve = tpc / (n_gt + 1e-16)\n            r.append(recall_curve[-1])\n\n            precision_curve = tpc / (tpc + fpc)\n            p.append(precision_curve[-1])\n\n            ap.append(compute_ap(recall_curve, precision_curve))\n\n    p, r, ap = np.array(p), np.array(r), np.array(ap)\n    f1 = 2 * p * r / (p + r + 1e-16)\n\n    return p, r, ap, f1, unique_classes.astype(\"int32\")\n\n\ndef compute_ap(recall, precision):\n    mrec = np.concatenate(([0.0], recall, [1.0]))\n    mpre = np.concatenate(([0.0], precision, [0.0]))\n    for i in range(mpre.size - 1, 0, -1):\n        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n    i = np.where(mrec[1:] != mrec[:-1])[0]\n    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n    return ap\n\n\ndef get_batch_statistics(outputs, targets, iou_threshold):\n    batch_metrics = []\n    for sample_i in range(len(outputs)):\n\n        if outputs[sample_i] is None:\n            continue\n\n        output = outputs[sample_i]\n        pred_boxes = output[:, :4]\n        pred_scores = output[:, 4]\n        pred_labels = output[:, -1]\n\n        true_positives = np.zeros(pred_boxes.shape[0])\n\n        annotations = targets[targets[:, 0] == sample_i][:, 1:]\n        target_labels = annotations[:, 0] if len(annotations) else []\n        if len(annotations):\n            detected_boxes = []\n            target_boxes = annotations[:, 1:]\n\n            for pred_i, (pred_box, pred_label) in enumerate(zip(pred_boxes, pred_labels)):\n\n                if len(detected_boxes) == len(annotations):\n                    break\n                if pred_label not in target_labels:\n                    continue\n\n                iou, box_index = bbox_iou(pred_box.unsqueeze(0), target_boxes).max(0)\n                if iou >= iou_threshold and box_index not in detected_boxes:\n                    true_positives[pred_i] = 1\n                    detected_boxes += [box_index]\n        batch_metrics.append([true_positives, pred_scores, pred_labels])\n    return batch_metrics\n\n\ndef bbox_wh_iou(wh1, wh2):\n    wh2 = wh2.t()\n    w1, h1 = wh1[0], wh1[1]\n    w2, h2 = wh2[0], wh2[1]\n    inter_area = torch.min(w1, w2) * torch.min(h1, h2)\n    union_area = (w1 * h1 + 1e-16) + w2 * h2 - inter_area\n    return inter_area / union_area\n\n\ndef bbox_iou(box1, box2, x1y1x2y2=True):\n    if not x1y1x2y2:\n        b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2\n        b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2\n        b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2\n        b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2\n    else:\n        b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]\n        b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]\n\n    inter_rect_x1 = torch.max(b1_x1, b2_x1)\n    inter_rect_y1 = torch.max(b1_y1, b2_y1)\n    inter_rect_x2 = torch.min(b1_x2, b2_x2)\n    inter_rect_y2 = torch.min(b1_y2, b2_y2)\n    inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(\n        inter_rect_y2 - inter_rect_y1 + 1, min=0\n    )\n    b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1)\n    b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1)\n\n    iou = inter_area / (b1_area + b2_area - inter_area + 1e-16)\n\n    return iou\n\n\ndef non_max_suppression(prediction, conf_thres=0.5, nms_thres=0.4):\n    prediction[..., :4] = xywh2xyxy(prediction[..., :4])\n    output = [None for _ in range(len(prediction))]\n    for image_i, image_pred in enumerate(prediction):\n        image_pred = image_pred[image_pred[:, 4] >= conf_thres]\n        if not image_pred.size(0):\n            continue\n        score = image_pred[:, 4] * image_pred[:, 5:].max(1)[0]\n        image_pred = image_pred[(-score).argsort()]\n        class_confs, class_preds = image_pred[:, 5:].max(1, keepdim=True)\n        detections = torch.cat((image_pred[:, :5], class_confs.float(), class_preds.float()), 1)\n        keep_boxes = []\n        while detections.size(0):\n            large_overlap = bbox_iou(detections[0, :4].unsqueeze(0), detections[:, :4]) > nms_thres\n            label_match = detections[0, -1] == detections[:, -1]\n            invalid = large_overlap & label_match\n            weights = detections[invalid, 4:5]\n            detections[0, :4] = (weights * detections[invalid, :4]).sum(0) / weights.sum()\n            keep_boxes += [detections[0]]\n            detections = detections[~invalid]\n        if keep_boxes:\n            output[image_i] = torch.stack(keep_boxes)\n\n    return output\n\n\ndef build_targets(pred_boxes, pred_cls, target, anchors, ignore_thres):\n\n    BoolTensor = torch.cuda.BoolTensor if pred_boxes.is_cuda else torch.BoolTensor\n    FloatTensor = torch.cuda.FloatTensor if pred_boxes.is_cuda else torch.FloatTensor\n\n    nB = pred_boxes.size(0)\n    nA = pred_boxes.size(1)\n    nC = pred_cls.size(-1)\n    nG = pred_boxes.size(2)\n\n    obj_mask = BoolTensor(nB, nA, nG, nG).fill_(0)\n    noobj_mask = BoolTensor(nB, nA, nG, nG).fill_(1)\n    class_mask = FloatTensor(nB, nA, nG, nG).fill_(0)\n    iou_scores = FloatTensor(nB, nA, nG, nG).fill_(0)\n    tx = FloatTensor(nB, nA, nG, nG).fill_(0)\n    ty = FloatTensor(nB, nA, nG, nG).fill_(0)\n    tw = FloatTensor(nB, nA, nG, nG).fill_(0)\n    th = FloatTensor(nB, nA, nG, nG).fill_(0)\n    tcls = FloatTensor(nB, nA, nG, nG, nC).fill_(0)\n\n    target_boxes = target[:, 2:6] * nG\n    gxy = target_boxes[:, :2]\n    gwh = target_boxes[:, 2:]\n    ious = torch.stack([bbox_wh_iou(anchor, gwh) for anchor in anchors])\n    best_ious, best_n = ious.max(0)\n    b, target_labels = target[:, :2].long().t()\n    gx, gy = gxy.t()\n    gw, gh = gwh.t()\n    gi, gj = gxy.long().t()\n    obj_mask[b, best_n, gj, gi] = 1\n    noobj_mask[b, best_n, gj, gi] = 0\n\n    for i, anchor_ious in enumerate(ious.t()):\n        noobj_mask[b[i], anchor_ious > ignore_thres, gj[i], gi[i]] = 0\n\n    tx[b, best_n, gj, gi] = gx - gx.floor()\n    ty[b, best_n, gj, gi] = gy - gy.floor()\n    tw[b, best_n, gj, gi] = torch.log(gw / anchors[best_n][:, 0] + 1e-16)\n    th[b, best_n, gj, gi] = torch.log(gh / anchors[best_n][:, 1] + 1e-16)\n    tcls[b, best_n, gj, gi, target_labels] = 1\n    class_mask[b, best_n, gj, gi] = (pred_cls[b, best_n, gj, gi].argmax(-1) == target_labels).float()\n    iou_scores[b, best_n, gj, gi] = bbox_iou(pred_boxes[b, best_n, gj, gi], target_boxes, x1y1x2y2=False)\n\n    tconf = obj_mask.float()\n    return iou_scores, class_mask, obj_mask, noobj_mask, tx, ty, tw, th, tcls, tconf\n\ndef horisontal_flip(images, targets):\n    images = torch.flip(images, [-1])\n    targets[:, 2] = 1 - targets[:, 2]\n    return images, targets\n\n#---------------------------------------------------\ndef pad_to_square(img, pad_value):\n    c, h, w = img.shape\n    dim_diff = np.abs(h - w)\n    pad1, pad2 = dim_diff // 2, dim_diff - dim_diff // 2\n    pad = (0, 0, pad1, pad2) if h <= w else (pad1, pad2, 0, 0)\n    img = F.pad(img, pad, \"constant\", value=pad_value)\n    return img, pad\n\n\ndef resize(image, size):\n    image = F.interpolate(image.unsqueeze(0), size=size, mode=\"nearest\").squeeze(0)\n    return image\n\n\ndef random_resize(images, min_size=288, max_size=448):\n    new_size = random.sample(list(range(min_size, max_size + 1, 32)), 1)[0]\n    images = F.interpolate(images, size=new_size, mode=\"nearest\")\n    return images\n\n\nclass ImageFolder(Dataset):\n    def __init__(self, folder_path, img_size=416):\n        self.files = sorted(glob.glob(\"%s/*.*\" % folder_path))\n        self.img_size = img_size\n\n    def __getitem__(self, index):\n        img_path = self.files[index % len(self.files)]\n        img = transforms.ToTensor()(Image.open(img_path))\n        img, _ = pad_to_square(img, 0)\n        img = resize(img, self.img_size)\n\n        return img_path, img\n\n    def __len__(self):\n        return len(self.files)\n\n\nclass ListDataset(Dataset):\n    def __init__(self, list_path, img_size=416, augment=True, multiscale=True, normalized_labels=True):\n        with open(list_path, \"r\") as file:\n            self.img_files = file.readlines()\n\n        self.label_files = [\n            path.replace(\"images\", \"labels\").replace(\".png\", \".txt\").replace(\".jpg\", \".txt\")\n            for path in self.img_files\n        ]\n        self.img_size = img_size\n        self.max_objects = 100\n        self.augment = augment\n        self.multiscale = multiscale\n        self.normalized_labels = normalized_labels\n        self.min_size = self.img_size - 3 * 32\n        self.max_size = self.img_size + 3 * 32\n        self.batch_count = 0\n\n    def __getitem__(self, index):\n        img_path = self.img_files[index % len(self.img_files)].rstrip()\n        img = transforms.ToTensor()(Image.open(img_path).convert('RGB'))\n        if len(img.shape) != 3:\n            img = img.unsqueeze(0)\n            img = img.expand((3, img.shape[1:]))\n\n        _, h, w = img.shape\n        h_factor, w_factor = (h, w) if self.normalized_labels else (1, 1)\n        img, pad = pad_to_square(img, 0)\n        _, padded_h, padded_w = img.shape\n\n        label_path = self.label_files[index % len(self.img_files)].rstrip()\n\n        targets = None\n        if os.path.exists(label_path):\n            boxes = torch.from_numpy(np.loadtxt(label_path).reshape(-1, 5))\n            # Extract coordinates for unpadded + unscaled image\n            x1 = w_factor * (boxes[:, 1] - boxes[:, 3] / 2)\n            y1 = h_factor * (boxes[:, 2] - boxes[:, 4] / 2)\n            x2 = w_factor * (boxes[:, 1] + boxes[:, 3] / 2)\n            y2 = h_factor * (boxes[:, 2] + boxes[:, 4] / 2)\n            # Adjust for added padding\n            x1 += pad[0]\n            y1 += pad[2]\n            x2 += pad[1]\n            y2 += pad[3]\n            # Returns (x, y, w, h)\n            boxes[:, 1] = ((x1 + x2) / 2) / padded_w\n            boxes[:, 2] = ((y1 + y2) / 2) / padded_h\n            boxes[:, 3] *= w_factor / padded_w\n            boxes[:, 4] *= h_factor / padded_h\n\n            targets = torch.zeros((len(boxes), 6))\n            targets[:, 1:] = boxes\n\n        if self.augment:\n            if np.random.random() < 0.5:\n                img, targets = horisontal_flip(img, targets)\n\n        return img_path, img, targets\n\n    def collate_fn(self, batch):\n        paths, imgs, targets = list(zip(*batch))\n        targets = [boxes for boxes in targets if boxes is not None]\n        for i, boxes in enumerate(targets):\n            boxes[:, 0] = i\n        targets = torch.cat(targets, 0)\n        if self.multiscale and self.batch_count % 10 == 0:\n            self.img_size = random.choice(range(self.min_size, self.max_size + 1, 32))\n        imgs = torch.stack([resize(img, self.img_size) for img in imgs])\n        self.batch_count += 1\n        return paths, imgs, targets\n\n    def __len__(self):\n        return len(self.img_files)\n\n# import MODELOS\ndef parse_model_config(path):\n    file = open(path, 'r')\n    lines = file.read().split('\\n')\n    lines = [x for x in lines if x and not x.startswith('#')]\n    lines = [x.rstrip().lstrip() for x in lines]\n    module_defs = []\n    for line in lines:\n        if line.startswith('['): \n            module_defs.append({})\n            module_defs[-1]['type'] = line[1:-1].rstrip()\n            if module_defs[-1]['type'] == 'convolutional':\n                module_defs[-1]['batch_normalize'] = 0\n        else:\n            key, value = line.split(\"=\")\n            value = value.strip()\n            module_defs[-1][key.rstrip()] = value.strip()\n\n    return module_defs\n\ndef parse_data_config(path):\n    options = dict()\n    options['gpus'] = '0,1,2,3'\n    options['num_workers'] = '10'\n    with open(path, 'r') as fp:\n        lines = fp.readlines()\n    for line in lines:\n        line = line.strip()\n        if line == '' or line.startswith('#'):\n            continue\n        key, value = line.split('=')\n        options[key.strip()] = value.strip()\n    return options\n\ndef create_modules(module_defs):\n    hyperparams = module_defs.pop(0)\n    output_filters = [int(hyperparams[\"channels\"])]\n    module_list = nn.ModuleList()\n    for module_i, module_def in enumerate(module_defs):\n        modules = nn.Sequential()\n\n        if module_def[\"type\"] == \"convolutional\":\n            bn = int(module_def[\"batch_normalize\"])\n            filters = int(module_def[\"filters\"])\n            kernel_size = int(module_def[\"size\"])\n            pad = (kernel_size - 1) // 2\n            modules.add_module(\n                f\"conv_{module_i}\",\n                nn.Conv2d(\n                    in_channels=output_filters[-1],\n                    out_channels=filters,\n                    kernel_size=kernel_size,\n                    stride=int(module_def[\"stride\"]),\n                    padding=pad,\n                    bias=not bn,\n                ),\n            )\n            if bn:\n                modules.add_module(f\"batch_norm_{module_i}\", nn.BatchNorm2d(filters, momentum=0.9, eps=1e-5))\n            if module_def[\"activation\"] == \"leaky\":\n                modules.add_module(f\"leaky_{module_i}\", nn.LeakyReLU(0.1))\n\n        elif module_def[\"type\"] == \"maxpool\":\n            kernel_size = int(module_def[\"size\"])\n            stride = int(module_def[\"stride\"])\n            if kernel_size == 2 and stride == 1:\n                modules.add_module(f\"_debug_padding_{module_i}\", nn.ZeroPad2d((0, 1, 0, 1)))\n            maxpool = nn.MaxPool2d(kernel_size=kernel_size, stride=stride, padding=int((kernel_size - 1) // 2))\n            modules.add_module(f\"maxpool_{module_i}\", maxpool)\n\n        elif module_def[\"type\"] == \"upsample\":\n            upsample = Upsample(scale_factor=int(module_def[\"stride\"]), mode=\"nearest\")\n            modules.add_module(f\"upsample_{module_i}\", upsample)\n\n        elif module_def[\"type\"] == \"route\":\n            layers = [int(x) for x in module_def[\"layers\"].split(\",\")]\n            filters = sum([output_filters[1:][i] for i in layers])\n            modules.add_module(f\"route_{module_i}\", EmptyLayer())\n\n        elif module_def[\"type\"] == \"shortcut\":\n            filters = output_filters[1:][int(module_def[\"from\"])]\n            modules.add_module(f\"shortcut_{module_i}\", EmptyLayer())\n\n        elif module_def[\"type\"] == \"yolo\":\n            anchor_idxs = [int(x) for x in module_def[\"mask\"].split(\",\")]\n            # Extract anchors\n            anchors = [int(x) for x in module_def[\"anchors\"].split(\",\")]\n            anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)]\n            anchors = [anchors[i] for i in anchor_idxs]\n            num_classes = int(module_def[\"classes\"])\n            img_size = int(hyperparams[\"height\"])\n            # Define detection layer\n            yolo_layer = YOLOLayer(anchors, num_classes, img_size)\n            modules.add_module(f\"yolo_{module_i}\", yolo_layer)\n        # Register module list and number of output filters\n        module_list.append(modules)\n        output_filters.append(filters)\n\n    return hyperparams, module_list\n\n\nclass Upsample(nn.Module):\n    def __init__(self, scale_factor, mode=\"nearest\"):\n        super(Upsample, self).__init__()\n        self.scale_factor = scale_factor\n        self.mode = mode\n\n    def forward(self, x):\n        x = F.interpolate(x, scale_factor=self.scale_factor, mode=self.mode)\n        return x\n\n\nclass EmptyLayer(nn.Module):\n    def __init__(self):\n        super(EmptyLayer, self).__init__()\n\n\nclass YOLOLayer(nn.Module):\n    def __init__(self, anchors, num_classes, img_dim=416):\n        super(YOLOLayer, self).__init__()\n        self.anchors = anchors\n        self.num_anchors = len(anchors)\n        self.num_classes = num_classes\n        self.ignore_thres = 0.5\n        self.mse_loss = nn.MSELoss()\n        self.bce_loss = nn.BCELoss()\n        self.obj_scale = 1\n        self.noobj_scale = 100\n        self.metrics = {}\n        self.img_dim = img_dim\n        self.grid_size = 0 \n\n    def compute_grid_offsets(self, grid_size, cuda=True):\n        self.grid_size = grid_size\n        g = self.grid_size\n        FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor\n        self.stride = self.img_dim / self.grid_size\n        self.grid_x = torch.arange(g).repeat(g, 1).view([1, 1, g, g]).type(FloatTensor)\n        self.grid_y = torch.arange(g).repeat(g, 1).t().view([1, 1, g, g]).type(FloatTensor)\n        self.scaled_anchors = FloatTensor([(a_w / self.stride, a_h / self.stride) for a_w, a_h in self.anchors])\n        self.anchor_w = self.scaled_anchors[:, 0:1].view((1, self.num_anchors, 1, 1))\n        self.anchor_h = self.scaled_anchors[:, 1:2].view((1, self.num_anchors, 1, 1))\n\n    def forward(self, x, targets=None, img_dim=None):\n        FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor\n        LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor\n        ByteTensor = torch.cuda.ByteTensor if x.is_cuda else torch.ByteTensor\n\n        self.img_dim = img_dim\n        num_samples = x.size(0)\n        grid_size = x.size(2)\n\n        prediction = (\n            x.view(num_samples, self.num_anchors, self.num_classes + 5, grid_size, grid_size)\n            .permute(0, 1, 3, 4, 2)\n            .contiguous()\n        )\n\n        x = torch.sigmoid(prediction[..., 0])  \n        y = torch.sigmoid(prediction[..., 1])  \n        w = prediction[..., 2]  \n        h = prediction[..., 3]  \n        pred_conf = torch.sigmoid(prediction[..., 4])  \n        pred_cls = torch.sigmoid(prediction[..., 5:])  \n\n        if grid_size != self.grid_size:\n            self.compute_grid_offsets(grid_size, cuda=x.is_cuda)\n\n        pred_boxes = FloatTensor(prediction[..., :4].shape)\n        pred_boxes[..., 0] = x.data + self.grid_x\n        pred_boxes[..., 1] = y.data + self.grid_y\n        pred_boxes[..., 2] = torch.exp(w.data) * self.anchor_w\n        pred_boxes[..., 3] = torch.exp(h.data) * self.anchor_h\n\n        output = torch.cat(\n            (\n                pred_boxes.view(num_samples, -1, 4) * self.stride,\n                pred_conf.view(num_samples, -1, 1),\n                pred_cls.view(num_samples, -1, self.num_classes),\n            ),\n            -1,\n        )\n\n        if targets is None:\n            return output, 0\n        else:\n            iou_scores, class_mask, obj_mask, noobj_mask, tx, ty, tw, th, tcls, tconf = build_targets(\n                pred_boxes=pred_boxes,\n                pred_cls=pred_cls,\n                target=targets,\n                anchors=self.scaled_anchors,\n                ignore_thres=self.ignore_thres,\n            )\n\n            loss_x = self.mse_loss(x[obj_mask], tx[obj_mask])\n            loss_y = self.mse_loss(y[obj_mask], ty[obj_mask])\n            loss_w = self.mse_loss(w[obj_mask], tw[obj_mask])\n            loss_h = self.mse_loss(h[obj_mask], th[obj_mask])\n            loss_conf_obj = self.bce_loss(pred_conf[obj_mask], tconf[obj_mask])\n            loss_conf_noobj = self.bce_loss(pred_conf[noobj_mask], tconf[noobj_mask])\n            loss_conf = self.obj_scale * loss_conf_obj + self.noobj_scale * loss_conf_noobj\n            loss_cls = self.bce_loss(pred_cls[obj_mask], tcls[obj_mask])\n            total_loss = loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls\n\n            cls_acc = 100 * class_mask[obj_mask].mean()\n            conf_obj = pred_conf[obj_mask].mean()\n            conf_noobj = pred_conf[noobj_mask].mean()\n            conf50 = (pred_conf > 0.5).float()\n            iou50 = (iou_scores > 0.5).float()\n            iou75 = (iou_scores > 0.75).float()\n            detected_mask = conf50 * class_mask * tconf\n            precision = torch.sum(iou50 * detected_mask) / (conf50.sum() + 1e-16)\n            recall50 = torch.sum(iou50 * detected_mask) / (obj_mask.sum() + 1e-16)\n            recall75 = torch.sum(iou75 * detected_mask) / (obj_mask.sum() + 1e-16)\n\n            self.metrics = {\n                \"loss\": to_cpu(total_loss).item(),\n                \"x\": to_cpu(loss_x).item(),\n                \"y\": to_cpu(loss_y).item(),\n                \"w\": to_cpu(loss_w).item(),\n                \"h\": to_cpu(loss_h).item(),\n                \"conf\": to_cpu(loss_conf).item(),\n                \"cls\": to_cpu(loss_cls).item(),\n                \"cls_acc\": to_cpu(cls_acc).item(),\n                \"recall50\": to_cpu(recall50).item(),\n                \"recall75\": to_cpu(recall75).item(),\n                \"precision\": to_cpu(precision).item(),\n                \"conf_obj\": to_cpu(conf_obj).item(),\n                \"conf_noobj\": to_cpu(conf_noobj).item(),\n                \"grid_size\": grid_size,\n            }\n\n            return output, total_loss\n\n\nclass Darknet(nn.Module):\n    def __init__(self, config_path, img_size=416):\n        super(Darknet, self).__init__()\n        self.module_defs = parse_model_config(config_path)\n        self.hyperparams, self.module_list = create_modules(self.module_defs)\n        self.yolo_layers = [layer[0] for layer in self.module_list if hasattr(layer[0], \"metrics\")]\n        self.img_size = img_size\n        self.seen = 0\n        self.header_info = np.array([0, 0, 0, self.seen, 0], dtype=np.int32)\n\n    def forward(self, x, targets=None):\n        img_dim = x.shape[2]\n        loss = 0\n        layer_outputs, yolo_outputs = [], []\n        for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n            if module_def[\"type\"] in [\"convolutional\", \"upsample\", \"maxpool\"]:\n                x = module(x)\n            elif module_def[\"type\"] == \"route\":\n                x = torch.cat([layer_outputs[int(layer_i)] for layer_i in module_def[\"layers\"].split(\",\")], 1)\n            elif module_def[\"type\"] == \"shortcut\":\n                layer_i = int(module_def[\"from\"])\n                x = layer_outputs[-1] + layer_outputs[layer_i]\n            elif module_def[\"type\"] == \"yolo\":\n                x, layer_loss = module[0](x, targets, img_dim)\n                loss += layer_loss\n                yolo_outputs.append(x)\n            layer_outputs.append(x)\n        yolo_outputs = to_cpu(torch.cat(yolo_outputs, 1))\n        return yolo_outputs if targets is None else (loss, yolo_outputs)\n\n    def load_darknet_weights(self, weights_path):\n        with open(weights_path, \"rb\") as f:\n            header = np.fromfile(f, dtype=np.int32, count=5)  \n            self.seen = header[3]  \n            weights = np.fromfile(f, dtype=np.float32)\n\n        cutoff = None\n        if \"darknet53.conv.74\" in weights_path:\n            cutoff = 75\n\n        ptr = 0\n        for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n            if i == cutoff:\n                break\n            if module_def[\"type\"] == \"convolutional\":\n                conv_layer = module[0]\n                if module_def[\"batch_normalize\"]:\n                    bn_layer = module[1]\n                    num_b = bn_layer.bias.numel()\n                    \n                    bn_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.bias)\n                    bn_layer.bias.data.copy_(bn_b)\n                    ptr += num_b\n                    bn_w = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.weight)\n                    bn_layer.weight.data.copy_(bn_w)\n                    ptr += num_b\n                    bn_rm = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_mean)\n                    bn_layer.running_mean.data.copy_(bn_rm)\n                    ptr += num_b\n                    bn_rv = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_var)\n                    bn_layer.running_var.data.copy_(bn_rv)\n                    ptr += num_b\n                else:\n                    num_b = conv_layer.bias.numel()\n                    conv_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(conv_layer.bias)\n                    conv_layer.bias.data.copy_(conv_b)\n                    ptr += num_b\n                num_w = conv_layer.weight.numel()\n                conv_w = torch.from_numpy(weights[ptr : ptr + num_w]).view_as(conv_layer.weight)\n                conv_layer.weight.data.copy_(conv_w)\n                ptr += num_w\n\n    def save_darknet_weights(self, path, cutoff=-1):\n        fp = open(path, \"wb\")\n        self.header_info[3] = self.seen\n        self.header_info.tofile(fp)\n\n        for i, (module_def, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n            if module_def[\"type\"] == \"convolutional\":\n                conv_layer = module[0]\n                if module_def[\"batch_normalize\"]:\n                    bn_layer = module[1]\n                    bn_layer.bias.data.cpu().numpy().tofile(fp)\n                    bn_layer.weight.data.cpu().numpy().tofile(fp)\n                    bn_layer.running_mean.data.cpu().numpy().tofile(fp)\n                    bn_layer.running_var.data.cpu().numpy().tofile(fp)\n                else:\n                    conv_layer.bias.data.cpu().numpy().tofile(fp)\n                conv_layer.weight.data.cpu().numpy().tofile(fp)\n\n        fp.close()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:44:21.386211Z","iopub.execute_input":"2021-11-11T20:44:21.386517Z","iopub.status.idle":"2021-11-11T20:44:21.509212Z","shell.execute_reply.started":"2021-11-11T20:44:21.386448Z","shell.execute_reply":"2021-11-11T20:44:21.508303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clase Cuadrante\nclase Cuadrante encaso de que solo el clasificador identidique x<4 cuadrantes se completa y corrige con sus funciones correspondientes ","metadata":{}},{"cell_type":"code","source":"class cuadrante:\n    lista=[]\n    cuadrante=[]\n    p=[]\n    h=0\n    w=0\n    def __init__(self,csi,csd,cii,cid,h,w):\n        self.h=h\n        self.w=w\n        self.cuadrante=np.zeros(4,dtype=int)\n        self.lista = np.zeros(16,dtype=int)\n\n        if len(csi)==4 and self.correcto(csi[3],csi[1])==1:\n            self.cuadrante[0]=1\n            self.lista[0] = csi[2]  # y1\n            self.lista[1] = csi[3]  # y2\n            self.lista[2] = csi[0]  # x1\n            self.lista[3] = csi[1]  # x2\n        if len(csd)==4 and self.correcto(csd[3],csd[1])==2:\n            self.cuadrante[1]=1\n            self.lista[4] = csd[2]  # y1\n            self.lista[5] = csd[3]  # y2\n            self.lista[6] = csd[0]  # x1\n            self.lista[7] = csd[1]  # x2\n        if len(cii)==4 and self.correcto(cii[3],cii[1])==3:\n            self.cuadrante[2]=1\n            self.lista[8] = cii[2]  # y1\n            self.lista[9] = cii[3]  # y2\n            self.lista[10] = cii[0]  # x1\n            self.lista[11] = cii[1]  # x2\n        if len(cid)==4 and self.correcto(cid[3],cid[1])==4:\n            self.cuadrante[3]=1\n            self.lista[12] = cid[2]  # y1\n            self.lista[13] = cid[3]  # y2\n            self.lista[14] = cid[0]  # x1\n            self.lista[15] = cid[1]  # x2\n\n    def correcto(self,h,w):\n        sector=0\n        #cuadrante 1\n        if ((h-self.h/4)<self.h/2) and ((w-self.w/4)<self.w/2):\n            sector=1\n        #cuadrante 2\n        if ((h-self.h/4)<self.h/2) and ((w-self.w/4)>self.w/2):\n            sector=2\n        #cuadrante 3\n        if((h-self.h/4)>self.h/2) and ((w-self.w/4)<self.w/2):\n            sector=3\n        # cuadrante 4\n        if ((h - self.h / 4) > self.h / 2) and ((w - self.w / 4) < self.w / 2):\n            sector=4\n\n        return sector\n\n    def completo(self):\n        return (np.prod(self.cuadrante),self.cuadrante)\n\n    def cuadranteCenter(self,index):\n        y=0\n        x=0\n        if index==0:\n            y = int((self.h + (2 * self.lista[1])) / 4)\n            x = int((self.w + (2 * self.lista[3])) / 4)\n        if index==1:\n            y = int((self.h + (2 * self.lista[5])) / 4)\n            x = int((self.w + (2 * self.lista[6])) / 4)\n        if index==2:\n            y = int((self.h + (2 * self.lista[8])) / 4)\n            x = int((self.w + (2 * self.lista[11])) / 4)\n        if index==3:\n            y = int((self.h + (2 * self.lista[12])) / 4)\n            x = int((self.w + (2 * self.lista[14])) / 4)\n        temp = [y, x]\n        self.p = temp\n\n    def cuadranteCompletar(self,x,y):\n        # en funcion de Cuadrante 0\n        if x==0 and y==1:\n            self.lista[4] = self.lista[0]\n            self.lista[5] = self.lista[1]\n            self.lista[6] = self.p[1] - 100\n            self.lista[7] = int((self.w + (2 * (self.lista[6] + self.lista[3] + self.lista[2]))) / 4)\n\n        if x==0 and y==2:\n            self.lista[8] = self.p[0] - 200\n            self.lista[9] = int((self.h / 2) + self.lista[8])\n            c = 0\n            if (self.w / 2) - self.lista[2] > 300:\n                c = -200\n            self.lista[10] = self.lista[2] + c\n            self.lista[11] = self.p[1] + 200\n\n        if x==0 and y==3:\n            self.lista[12] = self.p[0] - 100\n            self.lista[13] = int((self.h / 2) + self.lista[12])\n            self.lista[14] = self.p[1] - 100\n            self.lista[15] = int((self.w / 2) + self.lista[14])\n        # en funcion de Cuadrante 1 csd\n        if x==1 and y==0:\n            self.lista[0] = self.lista[4]\n            self.lista[1] = self.lista[5]\n            self.lista[3] = self.p[1] + 100\n            self.lista[2] = int((2 * (self.lista[7] - self.lista[6] - self.lista[3]) + self.w) / 4)\n\n        if x==1 and y==2:\n            self.lista[8] = self.p[0] - 100\n            self.lista[9] = int((self.h / 2) + self.lista[8]) - 200\n            c = 0\n            if (self.w / 2) - self.lista[2] > 300:\n                c = -200\n            self.lista[10] = self.lista[2] + c\n            # self.lista[10] =int((2*(self.lista[7]+self.lista[6]+self.lista[3])+self.w)/4)\n            self.lista[11] = self.p[1] + 100\n\n        if x==1 and y==3:\n            self.lista[12] = self.p[0] - 200\n            self.lista[13] = int((self.h / 2) + self.lista[12]) - 200\n            self.lista[14] = self.p[1] - 200\n            c = 0\n            if (self.w / 2) - self.lista[14] > 300:\n                c = -200\n            self.lista[15] = int((self.w / 2) + self.lista[14]) + c\n\n        # en funcion de Cuadrante 2 cii\n        if x==2 and y==0:\n            self.lista[1] = self.p[0] - 100\n            c = 0\n            t = int((2 * (self.lista[9] - self.lista[8] - self.lista[1]) + self.h) / 4)\n            if t > 500:\n                c = -300\n            self.lista[0] = t + c\n            c = 0\n            if ((self.w / 2) - self.lista[10] > 300):\n                c = -100\n            self.lista[2] = self.lista[10] - c\n            self.lista[3] = self.lista[11]\n\n        if x==2 and y==1:\n            self.lista[4] = self.lista[0]\n            self.lista[5] = self.p[0] + 100\n            self.lista[6] = self.p[1] - 100\n            self.lista[7] = int((self.w / 2) + self.lista[6]) - 100\n\n        if x==2 and y==3:\n            self.lista[12] = self.lista[8]\n            self.lista[13] = self.lista[9]\n            self.lista[14] = self.p[1] - 200\n            self.lista[15] = int((self.w / 2) + self.lista[14])\n\n        # en funcion de Cuadrante 3 cid\n        if x==3 and y==0:\n            self.lista[1] = self.p[0] + 100\n            self.lista[3] = self.p[1] + 100\n            t = int((2 * (self.lista[13] - self.lista[12] - self.lista[1]) + self.h) / 4)\n            c = 0\n            if t - 200 > 0:\n                c = -200\n            self.lista[0] = t + c\n            t = int((2 * (self.lista[15] - self.lista[14] - self.lista[3]) + self.w) / 4)\n            c = 0\n            if t - 300 > 0:\n                c + -300\n            self.lista[2] = t + c\n\n        if x==3 and y==1:\n            self.lista[5] = self.lista[1] - 100\n            self.lista[6] = self.lista[14]\n            self.lista[7] = self.lista[15]\n            # self.lista[4]=int((2*(self.lista[13]-self.lista[12]-self.lista[5])+self.h)/4)\n            self.lista[4] = self.lista[0]\n\n        if x==3 and y==2:\n            self.lista[8] = self.lista[12]\n            self.lista[9] = self.lista[13]\n            self.lista[11] = self.p[1] + 100\n            # self.lista[10] =int((2*(self.lista[15]-self.lista[14]-self.lista[11])+self.w)/4)\n            c = 0\n            if self.lista[2] - 200 > 0:\n                c = -200\n            self.lista[10] = self.lista[2] + c\n\n    def getLista(self):\n        completo=np.prod(self.cuadrante)\n        if completo == 0:\n            for l in range(0, len(self.cuadrante)):\n                if self.cuadrante[l] == 1 and completo == 0:\n                    self.cuadranteCenter(l)\n                    for index in range(0, len(self.cuadrante)):\n                        if l != index and self.cuadrante[index] == 0:\n                            self.cuadranteCompletar(l, index)\n                            self.cuadrante[index] = 1\n                completo = np.prod(self.cuadrante)\n\n        return self.lista","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:44:44.323518Z","iopub.execute_input":"2021-11-11T20:44:44.323802Z","iopub.status.idle":"2021-11-11T20:44:44.363177Z","shell.execute_reply.started":"2021-11-11T20:44:44.323769Z","shell.execute_reply":"2021-11-11T20:44:44.362511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Funciones y Procedimientos \nfunciones que permiten seleccionar los datos del data set siim-covid19-detection, y almacenarlos en una lista","metadata":{}},{"cell_type":"code","source":"#procedimiento que obtiene el nombre de la Imagen remplazando la cadena _image por .dcm\ndef imageLista(dfTrain,lista):\n    fila,columna=dfTrain.shape\n\n    for l in range(0,fila):\n        file=dfTrain.loc[l,'id']\n        id=dfTrain.loc[l,'StudyInstanceUID']\n        file=file.replace('_image','.dcm')\n        for l in lista:\n            if l[0]== id:\n                l[2]=file\n                break\n\n#procedimiento que optiene la segunda carpeta que contiene la imagen\ndef subCarpeta(path,lista):\n    #print(lista[1])\n    for subFile in lista:\n        aux= os.listdir(path+subFile[0])\n        for a in aux:\n            files=path+subFile[0]+'/'+a+'/'+subFile[2]\n            if os.path.isfile(files):\n                subFile[1]=a\n\n#funcion para mostrar resultados de listas \ndef mostrar(lista):\n    for l in lista:\n        print(l)\n        \n#funcion de Ruta permite rescatar la ruta de la imagen \ndef rutaLista(lista):\n    newLista=[]\n    for l in range(0,len(lista)):\n        newLista.append(lista[l][0]+'/'+lista[l][1]+'/'+lista[l][2])\n    \n    return newLista\n\n#funcion que debuelve una lista de imagenes del DataSet\ndef listaData(path,lista,x):\n    k=0\n    radiografia=[]\n    ruta='./norma/'\n    if x=='Covid':\n        ruta='./covid/'\n    namef=''\n    for l in lista:\n        #print(pathTrain+l)\n        ds=pydi.dcmread(path+l)\n        try:\n            d=np.array(ds.pixel_array)\n            namef=l[26:len(l)]\n            namef=namef.replace('.dcm','.jpg')\n            plt.imsave(ruta+namef,d)\n            temp=[ruta+namef,d.shape,0,0]\n            radiografia.append(temp)\n        except:\n            k+=1\n            \n    return radiografia\n\n#funcion que convierte la lista Aumentada de Datos a una lista Normal de entrenamiento\ndef listaDataAug(lista,listaDataAug):\n    tempLista=[]\n    for l in range(len(listaDataAug)):\n        #temp=[path+l,d.shape,0,0]\n        #temp=[listaCovid[listaDataAug[l][0]],listaCovid[listaDataAug[l][1]],listaCovid[listaDataAug[l][1]],0]\n        temp=[lista[listaDataAug[l][0]][0],lista[listaDataAug[l][0]][1],listaDataAug[l][1],1]\n        tempLista.append(temp)\n    return tempLista\n#------------------------------------------\ndef listenner(listaD,listaV):\n    for l in range(0,len(listaD)):\n        listaD[l][3]=1\n        \n    for l in listaV:\n        listaD[l][3]=0\n#------------------------------------------\ndef desvorde(lista):\n    for i in range(0,len(lista)):\n        if(lista[i]<0):\n            lista[i]=10\n    return lista\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:44:53.978929Z","iopub.execute_input":"2021-11-11T20:44:53.979785Z","iopub.status.idle":"2021-11-11T20:44:53.993242Z","shell.execute_reply.started":"2021-11-11T20:44:53.979748Z","shell.execute_reply":"2021-11-11T20:44:53.992619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def detector(listImg,k):\n    jcsi=[]\n    jcsd=[]\n    jcii=[]\n    jcid=[]\n    lista=[]\n    cont=0\n\n    for l in listImg:\n        imgRadio=cv2.imread(l[0])\n        h=imgRadio.shape[0]\n        w=imgRadio.shape[1]\n        #preguntar si es aumento \n        if l[3]==1:\n            if l[2]==1:\n                M = cv2.getRotationMatrix2D((w // 2, h // 2), 10, 1)\n            if l[2]==2:\n                M = cv2.getRotationMatrix2D((w // 2, h // 2), -10, 1)\n            if l[2]==3:\n                M = np.float32([[1, 0, 10], [0, 1, 0]])\n            if l[2]==4:\n                M = np.float32([[1, 0, -10], [0, 1, 0]])\n            if l[2]==5:\n                M = np.float32([[1, 0, 0], [0, 1, 10]])\n            if l[2]==6:\n                M = np.float32([[1, 0, 0], [0, 1, -10]])\n            if l[2]!=0 and l[3]!=0:    \n                imgRadio=cv2.warpAffine(imgRadio, M, (w, h))\n\n        RGBimg=convertiRgb(imgRadio)\n        imgTensor=transforms.ToTensor()(imgRadio)\n        imgTensor,_=pad_to_square(imgTensor,0)\n        imgTensor=resize(imgTensor,416)\n        imgTensor=imgTensor.unsqueeze(0)\n        imgTensor=Variable(imgTensor.type(Tensor))\n\n        detections=modelo(imgTensor)\n        detections=non_max_suppression(detections,confThres,0.4)\n        lista=[]\n        jcsi=[]\n        jcsd=[]\n        jcii=[]\n        jcid=[]\n        for detection in detections:\n            if detection is not None:\n                detection=rescale_boxes(detection,416, RGBimg.shape[:2])\n                for (x1,y1,x2,y2,conf,cls_conf,cls_pred)in detection:\n                    temp=[int(x1),int(x2),int(y1),int(y2)]\n                    if int(cls_pred)==0:\n                        jcsi=temp\n                    if int(cls_pred)==1:\n                        jcsd=temp\n                    if int(cls_pred)==2:\n                        jcii=temp\n                    if int(cls_pred)==3:\n                        jcid=temp\n\n        jcsi=desvorde(jcsi)\n        jcsd=desvorde(jcsd)\n        jcii=desvorde(jcii)\n        jcid=desvorde(jcid)\n\n        r=cuadrante(jcsi,jcsd,jcii,jcid,h,w)\n        lista=r.getLista()\n        lista=desvorde(lista)\n\n        obj1=imgRadio[lista[0]:lista[1],lista[2]:lista[3]]\n        obj2=imgRadio[lista[4]:lista[5],lista[6]:lista[7]]\n        obj3=imgRadio[lista[8]:lista[9],lista[10]:lista[11]]\n        obj4=imgRadio[lista[12]:lista[13],lista[14]:lista[15]]\n        \n        suma=np.sum(lista)\n        namef=l[0]\n        namef=namef[8:len(namef)]\n        if suma>0:\n            if l[3]==0:\n                if k=='Covid':\n                    plt.imsave('./torax/valid/jcsip/'+namef,obj1)\n                    plt.imsave('./torax/valid/jcsdp/'+namef,obj2)\n                    plt.imsave('./torax/valid/jciip/'+namef,obj3)\n                    plt.imsave('./torax/valid/jcidp/'+namef,obj4)\n                else: \n                    plt.imsave('./torax/valid/jcsin/'+namef,obj1)\n                    plt.imsave('./torax/valid/jcsdn/'+namef,obj2)\n                    plt.imsave('./torax/valid/jciin/'+namef,obj3)\n                    plt.imsave('./torax/valid/jcidn/'+namef,obj4)\n            else :\n                if k=='Covid':\n                    plt.imsave('./torax/train/jcsip/'+namef,obj1)\n                    plt.imsave('./torax/train/jcsdp/'+namef,obj2)\n                    plt.imsave('./torax/train/jciip/'+namef,obj3)\n                    plt.imsave('./torax/train/jcidp/'+namef,obj4)\n                else: \n                    plt.imsave('./torax/train/jcsip/'+namef,obj1)\n                    plt.imsave('./torax/train/jcsdp/'+namef,obj2)\n                    plt.imsave('./torax/train/jciip/'+namef,obj3)\n                    plt.imsave('./torax/train/jcidp/'+namef,obj4)\n        else:\n            #plt.imshow(imgRadio)\n            #plt.show()\n            print(cont,'Error cuadrante no detectado en {}'.format(k),l[0])\n            cont+=1\n            #print(lista)","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:02.414473Z","iopub.execute_input":"2021-11-11T20:45:02.414762Z","iopub.status.idle":"2021-11-11T20:45:02.434967Z","shell.execute_reply.started":"2021-11-11T20:45:02.414728Z","shell.execute_reply":"2021-11-11T20:45:02.433749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lectura de Archivos CSV","metadata":{}},{"cell_type":"code","source":"#lectura de carpetas contenedoras de imagenes y archivos CSV\npathTrain='../input/siim-covid19-detection/train/'\npathTest='../input/siim-covid19-detection/test/'\npathTrainCSV='../input/siim-covid19-detection/train_image_level.csv'\npathStudyCSV='../input/siim-covid19-detection/train_study_level.csv'\npathDataAug='../input/dataaug/DataAug.csv'\npathDataVali='../input/dataaug/DataValidation.csv'\npathDataValiN='../input/dataaug/DataValidationN.csv'","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:15.276505Z","iopub.execute_input":"2021-11-11T20:45:15.276757Z","iopub.status.idle":"2021-11-11T20:45:15.281974Z","shell.execute_reply.started":"2021-11-11T20:45:15.276729Z","shell.execute_reply":"2021-11-11T20:45:15.281205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Obtencion del size de los archivos \nverificacion de la cantidad de archivos presentes en los directorios","metadata":{}},{"cell_type":"code","source":"n=len(os.listdir(pathTrain))\nm=len(os.listdir(pathTest))\nprint('Numero de imagenes Train',n)\nprint('Numero de imagenes Test',m)","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:27.293079Z","iopub.execute_input":"2021-11-11T20:45:27.293841Z","iopub.status.idle":"2021-11-11T20:45:28.129264Z","shell.execute_reply.started":"2021-11-11T20:45:27.2938Z","shell.execute_reply":"2021-11-11T20:45:28.128413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lectura de Archivos CVS\nlectura de de los archhivos contenedores de informacion sobre etiquetas y hubicacion corespondiente","metadata":{}},{"cell_type":"code","source":"dfTrain=pd.read_csv(pathTrainCSV,sep=',')\ndfStudy=pd.read_csv(pathStudyCSV,sep=',')\ndfDataAug=pd.read_csv(pathDataAug,sep=',')\ndfDataValidation=pd.read_csv(pathDataVali,sep=',')\ndfDataValidationN=pd.read_csv(pathDataValiN,sep=',')\nprint('Lectura Terminada')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:36.305023Z","iopub.execute_input":"2021-11-11T20:45:36.305881Z","iopub.status.idle":"2021-11-11T20:45:36.413599Z","shell.execute_reply.started":"2021-11-11T20:45:36.305837Z","shell.execute_reply":"2021-11-11T20:45:36.412597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verificacion de Datos\nverificamos informacion de las dos listas en formato frame ","metadata":{}},{"cell_type":"code","source":"print(dfTrain.loc[:3,['id','StudyInstanceUID']])\nprint('------------------')\nprint(dfStudy.loc[:3,['id','Negative for Pneumonia','Atypical Appearance']])\nprint('------------------')\nprint(dfDataAug.loc[:3,['dcm','propiedad']])\nprint('------------------')\nprint(dfDataValidation.loc[:3,['index']])\nprint('------------------')\nprint(dfDataValidationN.loc[:3,['index']])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:44.396421Z","iopub.execute_input":"2021-11-11T20:45:44.396684Z","iopub.status.idle":"2021-11-11T20:45:44.425512Z","shell.execute_reply.started":"2021-11-11T20:45:44.396654Z","shell.execute_reply":"2021-11-11T20:45:44.424651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Estructura de Archivos\nse crea la estructura de archivos donde seran guardados los cuadrantes detectados para el entrenamiento, la estructura consta de las carpetas **train** y **valid** las mismas guardan  ['**jcsip**','**jcsin**','**jcsdp**','**jcsdn**','**jciip**','**jciin**','**jcidp**','**jcidn**'] en su interior","metadata":{}},{"cell_type":"code","source":"path='./torax/'\nif not os.path.exists(path):\n    os.mkdir(path)\n    os.mkdir('./covid')\n    os.mkdir('./norma')\n\ncarpeta=['train','valid']\nestructura=['jcsip','jcsin','jcsdp','jcsdn','jciip','jciin','jcidp','jcidn']\nfor c in carpeta:\n    if not os.path.exists(path+c):\n        os.mkdir(path+c)\n    for e in estructura:\n        if not os.path.exists(path+c+'/'+e):\n            os.mkdir(path+c+'/'+e)\n    \nfor p in os.listdir(path):\n    print(path+p)\n    for f in os.listdir(path+p):\n        print(' ',path+p+'/'+f)","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:45:57.207644Z","iopub.execute_input":"2021-11-11T20:45:57.208206Z","iopub.status.idle":"2021-11-11T20:45:57.221052Z","shell.execute_reply.started":"2021-11-11T20:45:57.208168Z","shell.execute_reply":"2021-11-11T20:45:57.220421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creacion de Listas Covid y Normal \nse crea 2 listas listaCovid y listaNormal, las mismas se forman apartir del frame dfStudy corespondiente al archivo '../input/siim-covid19-detection/train_study_level.csv', las listas cuentan con tres datos que almacenaran el **[id del dfStudy,sub carpeta,nombre archivo DICOM]**, al final se imprimen el len de las listas   ","metadata":{}},{"cell_type":"code","source":"listaCovid=[]\nlistaNormal=[]\nfila,columna=dfStudy.shape\n\nfor l in range(0,fila):\n    #print(dfStudy.loc[l,['id','Negative for Pneumonia','Atypical Appearance']])\n    ids=dfStudy.loc[l,'id']\n    ids=ids.replace('_study','')\n    if dfStudy.loc[l,'Negative for Pneumonia']== 1 :\n        listaNormal.append([ids,'',''])\n    if dfStudy.loc[l,'Atypical Appearance']== 1 :\n        listaCovid.append([ids,'',''])\n\nprint(len(listaCovid),len(listaNormal))","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:46:05.260885Z","iopub.execute_input":"2021-11-11T20:46:05.261603Z","iopub.status.idle":"2021-11-11T20:46:05.442465Z","shell.execute_reply.started":"2021-11-11T20:46:05.261555Z","shell.execute_reply":"2021-11-11T20:46:05.441664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listaDA=[]\nfila,columna=dfDataAug.shape\nfor l in range(0,fila):\n    temp=[dfDataAug.loc[l,'dcm'],dfDataAug.loc[l,'propiedad']]\n    listaDA.append(temp)\n\nlistaVA=[]\nfila,columna=dfDataValidation.shape\nfor l in range(0,fila):\n    listaVA.append(dfDataValidation.loc[l,'index'])\n\nlistaVAN=[]\nfila,columna=dfDataValidationN.shape\nfor l in range(0,fila):\n    listaVAN.append(dfDataValidationN.loc[l,'index'])\n    \nprint(listaDA[0],len(listaDA))\nprint(listaVA[0],len(listaVA))\nprint(listaVAN[0],len(listaVAN))","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:46:10.928662Z","iopub.execute_input":"2021-11-11T20:46:10.929358Z","iopub.status.idle":"2021-11-11T20:46:10.953607Z","shell.execute_reply.started":"2021-11-11T20:46:10.929313Z","shell.execute_reply":"2021-11-11T20:46:10.952586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Obtencion del nombre de la Imagen y lista con propiedades de entrenamiento\nse recorre el dataframe dfTrain para complementar y obtener el nombre del la imagen dicom, se remplaza la sub cadena **'_image'** por **'.dmc'** y se completa con la ruta de la sub carpeta y se crea la lista con propiedades para el entrenamiento de la red neurona, la lista tiene el formato de Lista=[**'ruta',(dimenciones),variavilidad,margen de error**]","metadata":{}},{"cell_type":"code","source":"#obtenemos el nombre de la Imagen\nimageLista(dfTrain,listaCovid)\nimageLista(dfTrain,listaNormal)\n\n#obtenemos la sub carpeta\nsubCarpeta(pathTrain,listaCovid)\nsubCarpeta(pathTrain,listaNormal)\n\n\n#obtenemos  la ruta completa\nlistaCovid1=rutaLista(listaCovid)\nlistaNormal1=rutaLista(listaNormal)\n\nprint('Rutas completas')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:46:19.312762Z","iopub.execute_input":"2021-11-11T20:46:19.313643Z","iopub.status.idle":"2021-11-11T20:46:26.0344Z","shell.execute_reply.started":"2021-11-11T20:46:19.313591Z","shell.execute_reply":"2021-11-11T20:46:26.033648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creamos la lista con propiedades para el entrenamiento\n\nlistaCovid=listaData(pathTrain,listaCovid1,'Covid')\nlistaNormal=listaData(pathTrain,listaNormal1,'Normal')\nprint(listaCovid[0])\nprint(listaNormal[0])\nprint('Ruta Terminada')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T20:46:37.826996Z","iopub.execute_input":"2021-11-11T20:46:37.827276Z","iopub.status.idle":"2021-11-11T21:11:28.027889Z","shell.execute_reply.started":"2021-11-11T20:46:37.827246Z","shell.execute_reply":"2021-11-11T21:11:28.026773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Lista Covid',len(listaCovid))\nprint('Lista Normal',len(listaNormal))","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:11:42.234857Z","iopub.execute_input":"2021-11-11T21:11:42.235251Z","iopub.status.idle":"2021-11-11T21:11:42.242571Z","shell.execute_reply.started":"2021-11-11T21:11:42.235187Z","shell.execute_reply":"2021-11-11T21:11:42.241628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aumento de Datos mediante archivo DataAug.csv creado por SMOTE\nse crea una lista con la funcion **listaDataAug(listaCovid,listaDA)**, y se adjunta la lista aumentada a la **listaCovid** creando **la listaFinal** ","metadata":{}},{"cell_type":"code","source":"#se crea la lista Aumentada de Datos\nlistaAumentada=listaDataAug(listaCovid,listaDA)\n#se combinan la listaCovid y listaAumentada en listafinal\nlistaFinalCovid=listaCovid+listaAumentada\nlistenner(listaFinalCovid,listaVA)\nlistenner(listaNormal,listaVAN)\nprint(len(listaFinalCovid),len(listaNormal))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:11:47.52305Z","iopub.execute_input":"2021-11-11T21:11:47.523334Z","iopub.status.idle":"2021-11-11T21:11:47.530676Z","shell.execute_reply.started":"2021-11-11T21:11:47.523301Z","shell.execute_reply":"2021-11-11T21:11:47.529679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# conteo de rangos no importante\nlista=listaFinal+listaNormal\nmilx=0\ndosmilx=0\ntresmilx=0\nmasx=0\nmily=0\ndosmily=0\ntresmily=0\nmasy=0\npx=0\npy=0\nfor l in lista:\n    x,y=l[1]\n    if x<1000:\n        milx+=1\n    if x>=1000 and x<2000:\n        dosmilx+=1\n    if x>=2000 and x<3000:\n        tresmilx+=1\n    if x>=3000:\n        masx+=1\n    if y<1000:\n        mily+=1\n    if y>=1000 and y<2000:\n        dosmily+=1\n    if y>=2000 and y<3000:\n        tresmily+=1\n    if y>=3000:\n        masy+=1\n    \n    px+=x\n    py+=y\n\npx=px/len(lista)\npy=py/len(lista)\n    \nprint(milx,dosmilx,tresmilx,masx)\nprint(mily,dosmily,tresmily,masy)\nprint(px,py)\nprint(len(lista))","metadata":{"execution":{"iopub.status.busy":"2021-11-10T14:48:58.256356Z","iopub.execute_input":"2021-11-10T14:48:58.2567Z","iopub.status.idle":"2021-11-10T14:48:58.274034Z","shell.execute_reply.started":"2021-11-10T14:48:58.256667Z","shell.execute_reply":"2021-11-10T14:48:58.273026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# verificacion de lectura de Radiografias\nverificamos que las listas contengan las rutas correctas de las Imagenes de Radiografias","metadata":{}},{"cell_type":"code","source":"#lista=np.random.choice(listaFinal,10)\ni=1\nfor r in range(0,10):\n    index=np.random.randint(0,len(listaFinalCovid))\n    print(listaFinalCovid[index])\n    img=cv2.imread(listaFinalCovid[index][0])\n    #ds=pydi.dcmread(listaFinal[index][0])\n    #rC=np.array(ds.pixel_array)\n    plt.subplot(2,5,i)\n    plt.imshow(img)\n    #plt.title(r)\n    i+=1\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:11:59.944063Z","iopub.execute_input":"2021-11-11T21:11:59.944801Z","iopub.status.idle":"2021-11-11T21:12:06.336941Z","shell.execute_reply.started":"2021-11-11T21:11:59.944761Z","shell.execute_reply":"2021-11-11T21:12:06.336047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creacion de Data de Entrenamiento y Data de Verificacion\n","metadata":{}},{"cell_type":"markdown","source":"# Detector de Cuadrantes","metadata":{}},{"cell_type":"code","source":"def convertiRgb(img):\n    b=img[:, :, 0].copy()\n    g=img[:, :, 1].copy()\n    r=img[:, :, 2].copy()\n\n    img[:, :, 0]=r\n    img[:, :, 1] =g\n    img[:, :, 2] =b\n\n    return img\n","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:12:13.627686Z","iopub.execute_input":"2021-11-11T21:12:13.62796Z","iopub.status.idle":"2021-11-11T21:12:13.633659Z","shell.execute_reply.started":"2021-11-11T21:12:13.627931Z","shell.execute_reply":"2021-11-11T21:12:13.632726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#archivos par el transfer learning\nweightPath='../input/dataaug/yolov3_ckpt_299.pth'\nmodelDef='../input/datamodelo/yolov3-custom.cfg'\nclassPath='../input/datamodelo/classes.names'\nconfThres=0.90","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:12:19.98879Z","iopub.execute_input":"2021-11-11T21:12:19.989332Z","iopub.status.idle":"2021-11-11T21:12:19.993777Z","shell.execute_reply.started":"2021-11-11T21:12:19.989298Z","shell.execute_reply":"2021-11-11T21:12:19.993056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seleccionamos dispositivo de calculo\n#device=torch.device('cpu')\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodelo=Darknet(modelDef,img_size=512).to(device)\nmodelo.load_state_dict(torch.load(weightPath))\nmodelo.eval()\n\nclasses=load_classes(classPath)\nTensor=torch.FloatTensor\ncolors=(0,0,255)","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:12:30.057009Z","iopub.execute_input":"2021-11-11T21:12:30.057702Z","iopub.status.idle":"2021-11-11T21:12:35.066741Z","shell.execute_reply.started":"2021-11-11T21:12:30.057661Z","shell.execute_reply":"2021-11-11T21:12:35.065913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detector de Cuadrante","metadata":{}},{"cell_type":"code","source":"print(listaFinalCovid[10])\nprint(listaNormal[10])","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:12:41.394667Z","iopub.execute_input":"2021-11-11T21:12:41.395203Z","iopub.status.idle":"2021-11-11T21:12:41.400802Z","shell.execute_reply.started":"2021-11-11T21:12:41.395152Z","shell.execute_reply":"2021-11-11T21:12:41.399877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detector(listaFinalCovid,'Covid')\nprint('cuadrantes Codiv Guardados')\n#detector(listaNormal,'Normal')\n#print('cuadrantes Normales Guardados')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:12:47.847434Z","iopub.execute_input":"2021-11-11T21:12:47.848216Z","iopub.status.idle":"2021-11-11T21:32:12.103639Z","shell.execute_reply.started":"2021-11-11T21:12:47.848137Z","shell.execute_reply":"2021-11-11T21:32:12.102642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detector(listaNormal,'Normal')\nprint('cuadrantes Normales Guardados')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T21:36:35.283323Z","iopub.execute_input":"2021-11-11T21:36:35.283663Z","iopub.status.idle":"2021-11-11T22:07:41.664982Z","shell.execute_reply.started":"2021-11-11T21:36:35.283612Z","shell.execute_reply":"2021-11-11T22:07:41.663891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = os.listdir('./torax/train/jcsip/')\nprint(l[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-11T22:23:58.036854Z","iopub.execute_input":"2021-11-11T22:23:58.037116Z","iopub.status.idle":"2021-11-11T22:23:58.043725Z","shell.execute_reply.started":"2021-11-11T22:23:58.03709Z","shell.execute_reply":"2021-11-11T22:23:58.042853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img=cv2.imread('./torax/train/jciip/'+l[0])\nplt.imshow(img)\nplt.show","metadata":{"execution":{"iopub.status.busy":"2021-11-11T22:27:21.499273Z","iopub.execute_input":"2021-11-11T22:27:21.500199Z","iopub.status.idle":"2021-11-11T22:27:22.220392Z","shell.execute_reply.started":"2021-11-11T22:27:21.500139Z","shell.execute_reply":"2021-11-11T22:27:22.219504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.make_archive('torax', 'zip', './torax/')","metadata":{"execution":{"iopub.status.busy":"2021-11-11T22:29:31.097915Z","iopub.execute_input":"2021-11-11T22:29:31.098654Z","iopub.status.idle":"2021-11-11T22:30:10.090303Z","shell.execute_reply.started":"2021-11-11T22:29:31.0986Z","shell.execute_reply":"2021-11-11T22:30:10.089369Z"},"trusted":true},"execution_count":null,"outputs":[]}]}