{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\npackage_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:31:51.03735Z","iopub.execute_input":"2021-06-01T04:31:51.037682Z","iopub.status.idle":"2021-06-01T04:31:51.046143Z","shell.execute_reply.started":"2021-06-01T04:31:51.037605Z","shell.execute_reply":"2021-06-01T04:31:51.045173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install -q \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install -q \"../input/hpacellsegmentatorraman/HPA-Cell-Segmentation\"","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:31:51.047857Z","iopub.execute_input":"2021-06-01T04:31:51.048442Z","iopub.status.idle":"2021-06-01T04:33:12.031002Z","shell.execute_reply.started":"2021-06-01T04:31:51.048382Z","shell.execute_reply":"2021-06-01T04:33:12.029953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\n\nalready_processed = pd.read_csv('../input/b7model/ensemble.csv')\nTEST = False\nprocessed_ids = set(already_processed.ID)\nprint(len(processed_ids))\nif len(os.listdir('../input/hpa-single-cell-image-classification/test')) == len(processed_ids) * 4:\n    print(\"In test mode\")\n    TEST = True\nprocessed_ids = set()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:12.034914Z","iopub.execute_input":"2021-06-01T04:33:12.035174Z","iopub.status.idle":"2021-06-01T04:33:14.508724Z","shell.execute_reply.started":"2021-06-01T04:33:12.035144Z","shell.execute_reply":"2021-06-01T04:33:14.507815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell\nimport time\nimport os\nimport sys\nimport cv2\nimport matplotlib.pyplot as plt\n\ndef chunks(lst, n):\n    for i in range(0, len(lst), n):\n        yield lst[i:i + n]\n\n\ndef process_single_image(file_name, rescaled=False, resized=True):\n    image = cv2.imread(file_name)[:, :, 0]\n    if resized:\n        image = cv2.resize(image, (512, 512))\n    if rescaled:\n        image = image / 255.0\n    return image\n\ndef process_folder(folder, model_folder, save_folder, prefix, num_in_chunk=24):\n    start = time.time()\n    NUC_MODEL = f'{model_folder}/dpn_unet_nuclei_v1.pth'\n    CELL_MODEL = f'{model_folder}/dpn_unet_cell_3ch_v1.pth'\n    segmentator = cellsegmentator.CellSegmentator(NUC_MODEL, CELL_MODEL, device='cuda', multi_channel_model=True)\n\n    file_name_all = list(set([x.split('_')[0] for x in os.listdir(folder)]))\n    file_name = [x for x in file_name_all if x not in processed_ids]\n    cell_dir = f'{save_folder}/{prefix}_cell_mask'\n    if not os.path.exists(cell_dir):\n        os.makedirs(cell_dir)\n    for chunk in chunks(file_name, num_in_chunk):\n        rcs = [process_single_image(f'{folder}/{stem}_red.png', True) for stem in chunk]\n        bcs = [process_single_image(f'{folder}/{stem}_blue.png', True) for stem in chunk]\n        ycs = [process_single_image(f'{folder}/{stem}_yellow.png', True) for stem in chunk]\n        rgb_batch = list(map(lambda x: np.stack(x, axis=2), zip(rcs, ycs, bcs)))\n        nuc_seg = segmentator.pred_nuclei(bcs)\n        cell_seg = segmentator.pred_cells(rgb_batch, precombined=True)\n\n        for i, stem in enumerate(chunk):\n            orig_size = cv2.imread(f'{folder}/{stem}_red.png').shape\n            nucl_mask, cell_mask = label_cell(nuc_seg[i], cell_seg[i])\n            cell_mask = cv2.resize(cell_mask, (orig_size[0], orig_size[1]), interpolation=cv2.INTER_NEAREST)\n            np.savez_compressed(f'{cell_dir}/{stem}', cell_mask)\n    end = time.time()\n    print(f'Segmentation took {end - start}')\n    return cell_dir","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:14.510324Z","iopub.execute_input":"2021-06-01T04:33:14.510821Z","iopub.status.idle":"2021-06-01T04:33:17.234314Z","shell.execute_reply.started":"2021-06-01T04:33:14.51078Z","shell.execute_reply":"2021-06-01T04:33:17.233493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str\n\ndef save_encoding(mask, name):\n    maxnum = np.max(mask[:, :])\n    lookup_image = {}\n    for i in range(1, maxnum + 1):\n        try:\n            cur_mask = (mask == i)\n            encoding = encode_binary_mask(cur_mask)\n            key = \"_\".join([name, str(i)])\n            lookup_image[key] = encoding\n        except:\n            continue\n    return lookup_image\n\ndef save_all_mask(folder, save_name):\n    mask_lookup = {}\n    for f in os.listdir(folder):\n        name = os.path.join(folder, f)\n        mask = np.load(name)['arr_0']\n        stem = f.split('.')[0]\n        im_lookup = save_encoding(mask, stem)\n        mask_lookup.update(im_lookup)\n    with open(save_name, 'wb') as f:\n        np.save(f, mask_lookup)\n    return mask_lookup","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:17.235565Z","iopub.execute_input":"2021-06-01T04:33:17.235914Z","iopub.status.idle":"2021-06-01T04:33:17.250599Z","shell.execute_reply.started":"2021-06-01T04:33:17.235878Z","shell.execute_reply":"2021-06-01T04:33:17.249727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import ceil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\n\ndef _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _array_feature(value):\n    bytes_image = value.tobytes()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[bytes_image]))\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef createExample(im_set, name, size, label=None):\n    r, c = size\n    if label is not None:\n        feature = {\n            'name': _bytes_feature(name.encode('utf-8')),\n            'label': _bytes_feature(label.encode('utf-8')),\n            'row': _int64_feature(r),\n            'col': _int64_feature(c),\n            'red': _array_feature(im_set[0]),\n            'blue': _array_feature(im_set[1]),\n            'yellow': _array_feature(im_set[2]),\n            'green': _array_feature(im_set[3])\n        }\n    else:\n        feature = {\n            'name': _bytes_feature(name.encode('utf-8')),\n            'row': _int64_feature(r),\n            'col': _int64_feature(c),\n            'red': _array_feature(im_set[0]),\n            'blue': _array_feature(im_set[1]),\n            'yellow': _array_feature(im_set[2]),\n            'green': _array_feature(im_set[3])\n        }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\ndef bbox2(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef make_square(h, w):\n    top, bottom, left, right = (0, 0, 0, 0)\n    if h > w:\n        left = (h-w)//2\n        right = (h-w)-left\n    if h < w:\n        top = (w-h)//2\n        bottom = (w-h)-top\n    return top, bottom, left, right\n\ndef calculate_new_size(row, col, max_size=250):\n    if row <= max_size and col <= max_size:\n        return row, col\n    a = 1\n    if row > col:\n        a = max_size * 1.0 / row\n        return max_size, int(col * a)\n    a = max_size * 1.0 / col\n    return int(a * row), max_size\n\ndef generate_cropped(writer, images, mask, stem, label=None):\n    for i in range(1, np.max(mask[:, :]) + 1):\n        cur = (mask == i)\n        rmin, rmax, cmin, cmax = bbox2(cur)\n        cropped_ims = []\n        try:\n            cropped_mask = cur[rmin:rmax, cmin:cmax]\n            new_size = calculate_new_size(rmax-rmin, cmax-cmin)\n            for im in images:\n                crop = im[rmin:rmax, cmin:cmax]\n                crop_masked = np.multiply(crop, cropped_mask)\n                crop_resized = cv2.resize(crop_masked, (new_size[1], new_size[0]))\n                cropped_ims.append(crop_resized)\n            new_name = stem + \"_\" + str(i)\n            ex = createExample(cropped_ims, new_name, new_size, label)\n            writer.write(ex.SerializeToString())\n        except:\n            continue\n\ndef process_data(df, name, train_folder, mask_folder, test=False):\n    label_lookup = dict(zip(list(df.ID), list(df.Label))) if not test else None\n    image_set = df['ID'] if not test else df\n    with tf.io.TFRecordWriter(name) as writer:\n        for imid in image_set:\n            red_im = cv2.imread(f'{train_folder}/{imid}_red.png')[:, :, 0]\n            blue_im = cv2.imread(f'{train_folder}/{imid}_blue.png')[:, :, 0]\n            green_im = cv2.imread(f'{train_folder}/{imid}_green.png')[:, :, 0]\n            yellow_im = cv2.imread(f'{train_folder}/{imid}_yellow.png')[:, :, 0]\n            mask = np.load(f'{mask_folder}/{imid}.npz')['arr_0']\n            label = label_lookup[imid] if not test else None\n            generate_cropped(writer, [red_im, blue_im, yellow_im, green_im], mask, imid, label)\n\n\ndef slice_to_tfrec(train_folder, mask_folder, collection, start_ind, name, test=False, num_in=2500, split_size=100):\n    start = time.time()\n    cur_slice = collection[start_ind:start_ind+num_in]\n    len_slice = len(cur_slice)\n    for i in range(ceil(len_slice/split_size)):\n        cur_name = f'{name}_{i}.tfrec'\n        print(cur_name)\n        process_data(cur_slice[i*split_size:(i+1)*split_size], cur_name, train_folder, mask_folder, test)\n        print(f'Done {i}')\n    end = time.time()\n    print(f'Create tfrec took {end - start}')","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:12:54.578395Z","iopub.execute_input":"2021-06-01T05:12:54.578737Z","iopub.status.idle":"2021-06-01T05:12:54.603069Z","shell.execute_reply.started":"2021-06-01T05:12:54.578684Z","shell.execute_reply":"2021-06-01T05:12:54.602256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB0, EfficientNetB1, EfficientNetB7, DenseNet121\nfrom tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras import Model\n\ndef b0_model(size):\n    base_model = EfficientNetB0(input_shape=(size, size,3), include_top=False, weights=None)\n    return base_model\n\ndef b7_model(size):\n    base_model = EfficientNetB7(input_shape=(size, size, 3), include_top=False, weights=None)\n    return base_model\n\ndef b1_model(size):\n    base_model = EfficientNetB1(input_shape=(size, size, 3), include_top=False, weights=None)\n    return base_model\n\ndef densenet_model(size):\n    base_model = DenseNet121(include_top=False, input_tensor=None, input_shape=(size, size, 3), weights=None)\n    return base_model\n\ndef make_model(base_model_fn, size, core_name, weight_folder):\n    base_model = base_model_fn(size)\n    base_model.trainable = False\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(19, activation='sigmoid')(x)\n    model = Model(inputs=base_model.input, outputs=predictions)\n    model.load_weights(f'{weight_folder}/{core_name}.h5')\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.025775Z","iopub.execute_input":"2021-06-01T04:33:22.026108Z","iopub.status.idle":"2021-06-01T04:33:22.036005Z","shell.execute_reply.started":"2021-06-01T04:33:22.026073Z","shell.execute_reply":"2021-06-01T04:33:22.035064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def record_to_test_v8(record):\n    record_format = {\n        'name': tf.io.FixedLenFeature([], tf.string),\n        'row': tf.io.FixedLenFeature([], tf.int64),\n        'col': tf.io.FixedLenFeature([], tf.int64),\n        'red': tf.io.FixedLenFeature([], tf.string),\n        'blue': tf.io.FixedLenFeature([], tf.string),\n        'yellow': tf.io.FixedLenFeature([], tf.string),\n        'green': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(record, record_format)\n    shape = (example['row'], example['col'])\n    rc = tf.reshape(tf.io.decode_raw(example['red'], out_type=tf.uint8), shape)\n    bc = tf.reshape(tf.io.decode_raw(example['blue'], out_type=tf.uint8), shape)\n    gc = tf.reshape(tf.io.decode_raw(example['green'], out_type=tf.uint8), shape)\n    im = tf.stack([rc, gc, bc], axis=2)\n    return im, example['name']\n\n\ndef preprocessing(size, image, name):\n    image = tf.image.resize_with_pad(image, size, size)\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.037234Z","iopub.execute_input":"2021-06-01T04:33:22.0376Z","iopub.status.idle":"2021-06-01T04:33:22.048716Z","shell.execute_reply.started":"2021-06-01T04:33:22.037562Z","shell.execute_reply":"2021-06-01T04:33:22.047921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_decoder_cell():\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_png(file_bytes, channels=3)\n        return img\n    return decode\n\ndef make_dataset(size, path):\n    dataset_name = tf.io.gfile.glob(path + '/test*.tfrec')\n    print(len(dataset_name))\n    dataset = tf.data.TFRecordDataset(dataset_name)\n    dataset = dataset.map(record_to_test_v8)\n    dataset = dataset.map(lambda x, y: preprocessing(size, x, y))\n    dataset = dataset.batch(1)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.0529Z","iopub.execute_input":"2021-06-01T04:33:22.053194Z","iopub.status.idle":"2021-06-01T04:33:22.061221Z","shell.execute_reply.started":"2021-06-01T04:33:22.05317Z","shell.execute_reply":"2021-06-01T04:33:22.060502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_decoder(target_size=(512, 512)):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_png(file_bytes, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    return decode\n\ndef build_dataset_full_size(paths):\n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    decoder = build_decoder()\n    dset = dset.map(decoder, num_parallel_calls=AUTO)\n    dset = dset.batch(1).prefetch(AUTO)\n    return dset\n\ndef predict_whole_image(folder, model, core_name):\n    # names = tf.io.gfile.glob(folder + '/*_green.png')\n    names = [f'{folder}/{x}_green.png' for x in image_set]\n    print(len(names))\n    test_set = build_dataset_full_size(names)\n    results = model.predict(test_set)\n    dictionary_name = f'{core_name}.npy'\n    lookup = {}\n    for ind, n in enumerate(names):\n        lookup[n] = results[ind, :]\n    with open(dictionary_name, 'wb') as f:\n        np.save(f, lookup)\n    return lookup","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.062486Z","iopub.execute_input":"2021-06-01T04:33:22.062746Z","iopub.status.idle":"2021-06-01T04:33:22.074866Z","shell.execute_reply.started":"2021-06-01T04:33:22.062718Z","shell.execute_reply":"2021-06-01T04:33:22.074104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ndef generate_submission_v2(lookup, mask, name, lookup_encoding):\n    maxnum = np.max(mask[:, :])\n    init_str = \"{},{},{},\".format(name, mask.shape[0], mask.shape[1])\n    for i in range(1, maxnum + 1):\n        key = \"_\".join([name, str(i)])\n        if key not in lookup.keys():\n            print(key + \" not found\")\n            continue\n        encoding = lookup_encoding[key]\n        pred = lookup[key]\n        for idx, p in enumerate(list(pred)):\n            pred_str = \"{} {} {} \".format(idx, p, encoding.decode())\n            init_str += pred_str\n    init_str += '\\n'\n    return init_str\n\ndef generate(folder, lookup_file, save_file, lookup_encoding):\n    fs = open(save_file, 'w')\n    fs.write('ID,ImageWidth,ImageHeight,PredictionString\\n')\n    lookup = np.load(lookup_file, allow_pickle=True).item()\n    for f in os.listdir(folder):\n        name = os.path.join(folder, f)\n        mask = np.load(name)['arr_0']\n        stem = f.split('.')[0]\n        s = generate_submission_v2(lookup, mask, stem, lookup_encoding)\n        fs.write(s)\n    fs.close()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.078018Z","iopub.execute_input":"2021-06-01T04:33:22.078264Z","iopub.status.idle":"2021-06-01T04:33:22.090014Z","shell.execute_reply.started":"2021-06-01T04:33:22.078241Z","shell.execute_reply":"2021-06-01T04:33:22.089172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def record_to_test_name_v8(record):\n    record_format = {\n        'name': tf.io.FixedLenFeature([], tf.string),\n        'row': tf.io.FixedLenFeature([], tf.int64),\n        'col': tf.io.FixedLenFeature([], tf.int64),\n        'red': tf.io.FixedLenFeature([], tf.string),\n        'blue': tf.io.FixedLenFeature([], tf.string),\n        'yellow': tf.io.FixedLenFeature([], tf.string),\n        'green': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(record, record_format)\n    return example['name']\n\ndef generate_name(path):\n    dataset_name = tf.io.gfile.glob(path + '/test*.tfrec')\n    dataset = tf.data.TFRecordDataset(dataset_name)\n    dataset = dataset.map(record_to_test_name_v8)\n    dataset_name_list = [y.numpy().decode('utf-8') for y in dataset]\n    print(len(dataset_name_list))\n    saved_name = 'name_list.npy'\n    np.save(saved_name, dataset_name_list)\n    return saved_name","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.092769Z","iopub.execute_input":"2021-06-01T04:33:22.093039Z","iopub.status.idle":"2021-06-01T04:33:22.102963Z","shell.execute_reply.started":"2021-06-01T04:33:22.093014Z","shell.execute_reply":"2021-06-01T04:33:22.1021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_to_dictionary(predictions, dataset, dictionary_name):\n    lookup = {}\n    for count, name in enumerate(dataset):\n        lookup[name] = predictions[count, :]\n    with open(dictionary_name, 'wb') as f:\n        np.save(f, lookup)\n    return lookup","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.105876Z","iopub.execute_input":"2021-06-01T04:33:22.106148Z","iopub.status.idle":"2021-06-01T04:33:22.112749Z","shell.execute_reply.started":"2021-06-01T04:33:22.106116Z","shell.execute_reply":"2021-06-01T04:33:22.111921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_and_save(core_name, attributes, dataset_name_list, lookup_encoding, cell_folder, weight_folder='../input/hpamodels'):\n    print(core_name)\n    start = time.time()\n    size = attributes['size']\n    model = make_model(attributes['model'], size, core_name, weight_folder)\n    dset = make_dataset(size, '/tmp')\n    predict_time = time.time()\n    results = model.predict(dset)\n    predict_time_end = time.time()\n    print(predict_time_end - predict_time)\n    dictionary_name = f'./{core_name}.npy'\n    lookup = convert_to_dictionary(results, dataset_name_list, dictionary_name)\n    # save_file = f'{core_name}.csv'\n    # generate(cell_folder, dictionary_name, save_file, lookup_encoding)\n    end = time.time()\n    print(end - start)\n    # return save_file","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.113846Z","iopub.execute_input":"2021-06-01T04:33:22.114262Z","iopub.status.idle":"2021-06-01T04:33:22.125362Z","shell.execute_reply.started":"2021-06-01T04:33:22.114159Z","shell.execute_reply":"2021-06-01T04:33:22.124467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ensemble(s, scores):\n    dfs = []\n    for prediction_file in s:\n        d = np.load(prediction_file, allow_pickle=True).item()\n        dfs.append(d)\n    newlookup = {}\n    for k in dfs[0].keys():\n        collected_pred = []\n        for p in dfs:\n            cur = p[k]\n            collected_pred.append(cur)\n        final_pred = np.zeros(19)\n        for ind, score in enumerate(scores):\n            final_pred += score * collected_pred[ind]\n        newlookup[k] = final_pred\n    with open('./ensemble.npy', 'wb') as f:\n        np.save(f, newlookup)\n    return newlookup","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.126517Z","iopub.execute_input":"2021-06-01T04:33:22.126888Z","iopub.status.idle":"2021-06-01T04:33:22.136028Z","shell.execute_reply.started":"2021-06-01T04:33:22.126854Z","shell.execute_reply":"2021-06-01T04:33:22.135209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def combine_whole_image(orig, image_pred, scores):\n    d = np.load(orig, allow_pickle=True).item()\n    image = np.load(image_pred, allow_pickle=True).item()\n    newlookup = {}\n    whole_im_lookup = {}\n    for key, val in image.items():\n        newkey = key.split('/')[-1].split('_')[0]\n        whole_im_lookup[newkey] = val\n    for k in d.keys():\n        final_pred = d[k] * scores[0]\n        image_key = k.split('_')[0]\n        im_pred = whole_im_lookup[image_key]\n        final_pred += scores[1] * im_pred\n        newlookup[k] = final_pred\n    with open('./ensemble_whole.npy', 'wb') as f:\n        np.save(f, newlookup)\n    return newlookup","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.137177Z","iopub.execute_input":"2021-06-01T04:33:22.137596Z","iopub.status.idle":"2021-06-01T04:33:22.146021Z","shell.execute_reply.started":"2021-06-01T04:33:22.137571Z","shell.execute_reply":"2021-06-01T04:33:22.145085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    cell_folder = process_folder('../input/hpa-single-cell-image-classification/test', '../input/hpacellsegmentatormodelweights', '/tmp', 'test')\n    mask_lookup = save_all_mask(cell_folder, './mask_lookup.npy')\n    image_set = [x.split('.')[0] for x in os.listdir(cell_folder)]\n    slice_to_tfrec('../input/hpa-single-cell-image-classification/test', cell_folder, image_set, 0, '/tmp/test', True, len(image_set))\n    dset = make_dataset(224, '/tmp')\n    dataset_name_list = generate_name('/tmp')\nexcept:\n    print(\"Preprocessing failed\")","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:33:22.147739Z","iopub.execute_input":"2021-06-01T04:33:22.148268Z","iopub.status.idle":"2021-06-01T04:47:03.064084Z","shell.execute_reply.started":"2021-06-01T04:33:22.148229Z","shell.execute_reply":"2021-06-01T04:47:03.063207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"core_name_dict = {\n                  'model-121-rgb-tpu-aug-kfold-0-smooth-255-224-v8': {'size': 224, 'model': densenet_model},\n                  'model-b1-rgb-tpu-aug-kfold-0-smooth-255-v8-224': {'size': 224, 'model': b1_model},\n                  }\n\ntry:\n    dataset_name_list = np.load('./name_list.npy')\n    lookup_encoding = np.load('./mask_lookup.npy', allow_pickle=True).item()\n    for name, attrs in core_name_dict.items():\n        predict_and_save(name, attrs, dataset_name_list, lookup_encoding, cell_folder)\n    first_ensemble = ensemble(['model-b1-rgb-tpu-aug-kfold-0-smooth-255-v8-224.npy', 'model-121-rgb-tpu-aug-kfold-0-smooth-255-224-v8.npy'],\n                              [0.5, 0.5])\nexcept:\n    print(\"Prediction cell failed\")","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:17:39.660355Z","iopub.execute_input":"2021-06-01T05:17:39.660676Z","iopub.status.idle":"2021-06-01T05:22:08.702673Z","shell.execute_reply.started":"2021-06-01T05:17:39.660647Z","shell.execute_reply":"2021-06-01T05:22:08.701813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    model_full = make_model(b7_model, 512, 'full-image-green-remaining', '../input/hpamodels')\n    model_full_lookup = predict_whole_image('../input/hpa-single-cell-image-classification/test', model_full, 'full-image-green-remaining')\n    final_ensemble = combine_whole_image('./ensemble.npy', 'full-image-green-remaining.npy', [0.8, 0.2])\nexcept:\n    print(\"Prediction image failed\")","metadata":{"execution":{"iopub.status.busy":"2021-06-01T04:47:03.075727Z","iopub.execute_input":"2021-06-01T04:47:03.076073Z","iopub.status.idle":"2021-06-01T04:48:33.191613Z","shell.execute_reply.started":"2021-06-01T04:47:03.076036Z","shell.execute_reply":"2021-06-01T04:48:33.190706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_file = 'submission_others.csv'\ntry:\n    generate(cell_folder, './ensemble_whole.npy', save_file, lookup_encoding)\nexcept:\n    print(\"Saving prediction failed\")","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:22:22.029684Z","iopub.execute_input":"2021-06-01T05:22:22.030145Z","iopub.status.idle":"2021-06-01T05:22:34.519881Z","shell.execute_reply.started":"2021-06-01T05:22:22.030104Z","shell.execute_reply":"2021-06-01T05:22:34.519003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    final_submission = pd.read_csv(save_file)\nexcept:\n    print(\"Failed to read\")\n    final_submission = already_processed\n# final_submission = pd.concat([other_submission, already_processed], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:22:42.213861Z","iopub.execute_input":"2021-06-01T05:22:42.214192Z","iopub.status.idle":"2021-06-01T05:22:42.75789Z","shell.execute_reply.started":"2021-06-01T05:22:42.214163Z","shell.execute_reply":"2021-06-01T05:22:42.757011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')\ndata = final_submission[['ID', 'PredictionString']]\ndata = data.rename(columns={'PredictionString': 'pred'})\n\nsub = pd.merge(\n    sample,\n    data,\n    how=\"left\",\n    left_on='ID',\n    right_on='ID',\n)\n\nprint(sub.head())\n\ndef isNaN(num):\n    return num != num\n\nfor i, row in sub.iterrows():\n    if isNaN(row['pred']): continue\n    sub.PredictionString.loc[i] = row['pred']\n\nsub = sub[sample.columns]\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:22:45.866402Z","iopub.execute_input":"2021-06-01T05:22:45.866751Z","iopub.status.idle":"2021-06-01T05:22:48.400901Z","shell.execute_reply.started":"2021-06-01T05:22:45.866713Z","shell.execute_reply":"2021-06-01T05:22:48.399995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n\nimages_dir_path = '/tmp'\n\ntry:\n    shutil.rmtree(images_dir_path)\nexcept OSError as e:\n    print(e)","metadata":{},"execution_count":null,"outputs":[]}]}