{
  "id": 169806,
  "title": "Image and Meta Data Extraction from tfrecords",
  "url": "/competitions/siim-isic-melanoma-classification/discussion/169806",
  "author_name": "",
  "post_date": "2020-07-25T09:58:29.533633800Z",
  "votes": 6,
  "comment_count": 4,
  "views": 0,
  "content": "<p>If you are not comfortable using tfrecord files, you can extract image and meta information from these files using <a href=\"https://github.com/vahidk/tfrecord\">TFRecord reader</a>.  Here is a sample code:</p>\n\n<p>` <br>\n    import os\n    import warnings\n    warnings.filterwarnings(\"ignore\", category=UserWarning)\n    import sys\n    import pandas as pd\n    from tqdm import tqdm as T\n    import torch\n    from tfrecord.torch.dataset import TFRecordDataset\n    import cv2\n    import gc\n    from p_tqdm import p_map    </p>\n\n<pre><code>data_dir = 'data'\ntfrec_dir = f'{data_dir}/tfrecords'\ntrain_dir = f\"{data_dir}/train\"\ntest_dir = f\"{data_dir}/test\"\nfilelist = os.listdir(tfrec_dir)\nfilelist = [f for f in filelist if '.tfrec' in f]\nos.makedirs(train_dir, exist_ok=True)\nos.makedirs(test_dir, exist_ok=True)\ntrain = pd.DataFrame()\ntest = pd.DataFrame()\n\ntrain_row = 0\ntest_row = 0\n\ndef decode_image(features):\n    # get BGR image from bytes\n    features[\"image\"] = cv2.imdecode(features[\"image\"], -1)\n    return features\n\n# Creating index files\nfor i in T(filelist):\n    tfrec_file = os.path.join(tfrec_dir, i)\n    index_file = os.path.join(tfrec_dir, i).replace('.tfrec', '.index')\n    if not os.path.exists(index_file):\n        os.system(f\"python3 -m tfrecord.tools.tfrecord2idx {tfrec_file} {index_file}\")\n\ndef tfrec_extract(filename):\n    global train_row\n    global test_row\n    global train\n    global test\n    tfrecord_path = os.path.join(tfrec_dir, filename)\n    index_path = tfrecord_path.replace('.tfrec', '.index')\n\n    if 'train' in filename:\n        savedir = train_dir\n    else:savedir = test_dir\n    dataset = TFRecordDataset(tfrecord_path, index_path, transform=decode_image)\n    loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n    for data in T(loader):\n        if 'train' in filename:\n            train_row += 1\n        else: test_row += 1\n        img_name = data['image_name'].squeeze().data.cpu().numpy().copy()\n        img_name = os.path.join(savedir, ''.join(map(chr, img_name)))\n        img_name += '.jpg'\n        image_file = data['image'].squeeze().data.cpu().numpy()\n        cv2.imwrite(img_name, image_file)\n        del data['image']\n        del data['image_name']\n        for k, v in data.items():\n            if 'train' in filename:\n                train.loc[train_row, 'image_name'] = img_name\n                train.loc[train_row, k] = v.squeeze().data.cpu().numpy()\n                train.loc[train_row, 'tfrec'] = filename.replace('.tfrec', '')\n            else:\n                test.loc[test_row, 'image_name'] = img_name\n                test.loc[test_row, k] = v.squeeze().data.cpu().numpy()\n                test.loc[test_row, 'tfrec'] = filename.replace('.tfrec', '')\n\n\n# p_map(tfrec_extract, filelist)\nfor f in T(filelist):\n    tfrec_extract(f)\n\ntrain.to_csv(f\"{data_dir}/train_768.csv\", index=False)\ntest.to_csv(f\"{data_dir}/test_768.csv\", index=False)\nprint(f\"total {train_row} train images and {test_row} test images\")\n</code></pre>\n\n<p>`</p>",
  "messages": [
    {
      "id": "944733",
      "postDate": "07/25/2020 09:58:29",
      "content": "<p>If you are not comfortable using tfrecord files, you can extract image and meta information from these files using <a href=\"https://github.com/vahidk/tfrecord\">TFRecord reader</a>.  Here is a sample code:</p>\n\n<p>` <br>\n    import os\n    import warnings\n    warnings.filterwarnings(\"ignore\", category=UserWarning)\n    import sys\n    import pandas as pd\n    from tqdm import tqdm as T\n    import torch\n    from tfrecord.torch.dataset import TFRecordDataset\n    import cv2\n    import gc\n    from p_tqdm import p_map    </p>\n\n<pre><code>data_dir = 'data'\ntfrec_dir = f'{data_dir}/tfrecords'\ntrain_dir = f\"{data_dir}/train\"\ntest_dir = f\"{data_dir}/test\"\nfilelist = os.listdir(tfrec_dir)\nfilelist = [f for f in filelist if '.tfrec' in f]\nos.makedirs(train_dir, exist_ok=True)\nos.makedirs(test_dir, exist_ok=True)\ntrain = pd.DataFrame()\ntest = pd.DataFrame()\n\ntrain_row = 0\ntest_row = 0\n\ndef decode_image(features):\n    # get BGR image from bytes\n    features[\"image\"] = cv2.imdecode(features[\"image\"], -1)\n    return features\n\n# Creating index files\nfor i in T(filelist):\n    tfrec_file = os.path.join(tfrec_dir, i)\n    index_file = os.path.join(tfrec_dir, i).replace('.tfrec', '.index')\n    if not os.path.exists(index_file):\n        os.system(f\"python3 -m tfrecord.tools.tfrecord2idx {tfrec_file} {index_file}\")\n\ndef tfrec_extract(filename):\n    global train_row\n    global test_row\n    global train\n    global test\n    tfrecord_path = os.path.join(tfrec_dir, filename)\n    index_path = tfrecord_path.replace('.tfrec', '.index')\n\n    if 'train' in filename:\n        savedir = train_dir\n    else:savedir = test_dir\n    dataset = TFRecordDataset(tfrecord_path, index_path, transform=decode_image)\n    loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n    for data in T(loader):\n        if 'train' in filename:\n            train_row += 1\n        else: test_row += 1\n        img_name = data['image_name'].squeeze().data.cpu().numpy().copy()\n        img_name = os.path.join(savedir, ''.join(map(chr, img_name)))\n        img_name += '.jpg'\n        image_file = data['image'].squeeze().data.cpu().numpy()\n        cv2.imwrite(img_name, image_file)\n        del data['image']\n        del data['image_name']\n        for k, v in data.items():\n            if 'train' in filename:\n                train.loc[train_row, 'image_name'] = img_name\n                train.loc[train_row, k] = v.squeeze().data.cpu().numpy()\n                train.loc[train_row, 'tfrec'] = filename.replace('.tfrec', '')\n            else:\n                test.loc[test_row, 'image_name'] = img_name\n                test.loc[test_row, k] = v.squeeze().data.cpu().numpy()\n                test.loc[test_row, 'tfrec'] = filename.replace('.tfrec', '')\n\n\n# p_map(tfrec_extract, filelist)\nfor f in T(filelist):\n    tfrec_extract(f)\n\ntrain.to_csv(f\"{data_dir}/train_768.csv\", index=False)\ntest.to_csv(f\"{data_dir}/test_768.csv\", index=False)\nprint(f\"total {train_row} train images and {test_row} test images\")\n</code></pre>\n\n<p>`</p>",
      "rawMarkdown": "If you are not comfortable using tfrecord files, you can extract image and meta information from these files using [TFRecord reader](https://github.com/vahidk/tfrecord).  Here is a sample code:\n\n`\t\n\timport os\n\timport warnings\n\twarnings.filterwarnings(\"ignore\", category=UserWarning)\n\timport sys\n\timport pandas as pd\n\tfrom tqdm import tqdm as T\n\timport torch\n\tfrom tfrecord.torch.dataset import TFRecordDataset\n\timport cv2\n\timport gc\n\tfrom p_tqdm import p_map\t\n\t\n\tdata_dir = 'data'\n\ttfrec_dir = f'{data_dir}/tfrecords'\n\ttrain_dir = f\"{data_dir}/train\"\n\ttest_dir = f\"{data_dir}/test\"\n\tfilelist = os.listdir(tfrec_dir)\n\tfilelist = [f for f in filelist if '.tfrec' in f]\n\tos.makedirs(train_dir, exist_ok=True)\n\tos.makedirs(test_dir, exist_ok=True)\n\ttrain = pd.DataFrame()\n\ttest = pd.DataFrame()\n\n\ttrain_row = 0\n\ttest_row = 0\n\n\tdef decode_image(features):\n\t    # get BGR image from bytes\n\t    features[\"image\"] = cv2.imdecode(features[\"image\"], -1)\n\t    return features\n\t\n\t# Creating index files\n\tfor i in T(filelist):\n\t    tfrec_file = os.path.join(tfrec_dir, i)\n\t    index_file = os.path.join(tfrec_dir, i).replace('.tfrec', '.index')\n\t    if not os.path.exists(index_file):\n\t        os.system(f\"python3 -m tfrecord.tools.tfrecord2idx {tfrec_file} {index_file}\")\n\n\tdef tfrec_extract(filename):\n\t    global train_row\n\t    global test_row\n\t    global train\n\t    global test\n\t    tfrecord_path = os.path.join(tfrec_dir, filename)\n\t    index_path = tfrecord_path.replace('.tfrec', '.index')\n\t    \n\t    if 'train' in filename:\n\t        savedir = train_dir\n\t    else:savedir = test_dir\n\t    dataset = TFRecordDataset(tfrecord_path, index_path, transform=decode_image)\n\t    loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n\t    for data in T(loader):\n\t        if 'train' in filename:\n\t            train_row += 1\n\t        else: test_row += 1\n\t        img_name = data['image_name'].squeeze().data.cpu().numpy().copy()\n\t        img_name = os.path.join(savedir, ''.join(map(chr, img_name)))\n\t        img_name += '.jpg'\n\t        image_file = data['image'].squeeze().data.cpu().numpy()\n\t        cv2.imwrite(img_name, image_file)\n\t        del data['image']\n\t        del data['image_name']\n\t        for k, v in data.items():\n\t            if 'train' in filename:\n\t                train.loc[train_row, 'image_name'] = img_name\n\t                train.loc[train_row, k] = v.squeeze().data.cpu().numpy()\n\t                train.loc[train_row, 'tfrec'] = filename.replace('.tfrec', '')\n\t            else:\n\t                test.loc[test_row, 'image_name'] = img_name\n\t                test.loc[test_row, k] = v.squeeze().data.cpu().numpy()\n\t                test.loc[test_row, 'tfrec'] = filename.replace('.tfrec', '')\n\n\n\t# p_map(tfrec_extract, filelist)\n\tfor f in T(filelist):\n\t    tfrec_extract(f)\n\n\ttrain.to_csv(f\"{data_dir}/train_768.csv\", index=False)\n\ttest.to_csv(f\"{data_dir}/test_768.csv\", index=False)\n\tprint(f\"total {train_row} train images and {test_row} test images\")\n`",
      "votes": null
    },
    {
      "id": "945169",
      "postDate": "07/25/2020 16:02:07",
      "content": "<p>Bhaia It would be better if you may share the the custom dataset that you created as \"TFRecordDataset \" .I am having error like \"no module named tfrecord\"</p>",
      "rawMarkdown": "Bhaia It would be better if you may share the the custom dataset that you created as \"TFRecordDataset \" .I am having error like \"no module named tfrecord\"",
      "votes": null
    },
    {
      "id": "945206",
      "postDate": "07/25/2020 16:42:11",
      "content": "<p>My internet speed is very slow right now. :( I'll try to upload it once it gets stable. However, you can install <code>tfrecord</code> with this command: <code>pip3 install tfrecord</code></p>",
      "rawMarkdown": "My internet speed is very slow right now. :( I'll try to upload it once it gets stable. However, you can install `tfrecord` with this command: `pip3 install tfrecord`",
      "votes": null
    },
    {
      "id": "945281",
      "postDate": "07/25/2020 17:47:23",
      "content": "<p>Okay bhai</p>",
      "rawMarkdown": "Okay bhai",
      "votes": null
    },
    {
      "id": "947841",
      "postDate": "07/27/2020 14:07:07",
      "content": "<p><a href=\"/tawheedrony\">@tawheedrony</a> I've uploaded the dataset here. This dataset is created by combining <a href=\"https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164092\">2020</a> and <a href=\"https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910\">2019-2017</a> competition datasets provided by chris. </p>",
      "rawMarkdown": "tawheedrony I've uploaded the dataset [here](www.kaggle.com/dataset/c1098e246be309c3f9781a78c372469a3bd7d51c2d963c30e1ebcd46d28bc70d). This dataset is created by combining [2020](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164092) and [2019-2017](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910) competition datasets provided by chris.",
      "votes": null
    }
  ],
  "comments": [
    {
      "id": 945169,
      "author_name": "tawheedrony",
      "author_url": "",
      "post_date": "07/25/2020 16:02:07",
      "content": "<p>Bhaia It would be better if you may share the the custom dataset that you created as \"TFRecordDataset \" .I am having error like \"no module named tfrecord\"</p>",
      "votes": null,
      "replies": [
        {
          "id": 945206,
          "author_name": "tahsin",
          "author_url": "",
          "post_date": "07/25/2020 16:42:11",
          "content": "<p>My internet speed is very slow right now. :( I'll try to upload it once it gets stable. However, you can install <code>tfrecord</code> with this command: <code>pip3 install tfrecord</code></p>",
          "votes": null,
          "replies": []
        },
        {
          "id": 945281,
          "author_name": "tawheedrony",
          "author_url": "",
          "post_date": "07/25/2020 17:47:23",
          "content": "<p>Okay bhai</p>",
          "votes": null,
          "replies": []
        },
        {
          "id": 947841,
          "author_name": "tahsin",
          "author_url": "",
          "post_date": "07/27/2020 14:07:07",
          "content": "<p><a href=\"/tawheedrony\">@tawheedrony</a> I've uploaded the dataset here. This dataset is created by combining <a href=\"https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164092\">2020</a> and <a href=\"https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910\">2019-2017</a> competition datasets provided by chris. </p>",
          "votes": null,
          "replies": []
        }
      ]
    }
  ],
  "raw_markdown_by_id": {
    "944733": "If you are not comfortable using tfrecord files, you can extract image and meta information from these files using [TFRecord reader](https://github.com/vahidk/tfrecord).  Here is a sample code:\n\n`\t\n\timport os\n\timport warnings\n\twarnings.filterwarnings(\"ignore\", category=UserWarning)\n\timport sys\n\timport pandas as pd\n\tfrom tqdm import tqdm as T\n\timport torch\n\tfrom tfrecord.torch.dataset import TFRecordDataset\n\timport cv2\n\timport gc\n\tfrom p_tqdm import p_map\t\n\t\n\tdata_dir = 'data'\n\ttfrec_dir = f'{data_dir}/tfrecords'\n\ttrain_dir = f\"{data_dir}/train\"\n\ttest_dir = f\"{data_dir}/test\"\n\tfilelist = os.listdir(tfrec_dir)\n\tfilelist = [f for f in filelist if '.tfrec' in f]\n\tos.makedirs(train_dir, exist_ok=True)\n\tos.makedirs(test_dir, exist_ok=True)\n\ttrain = pd.DataFrame()\n\ttest = pd.DataFrame()\n\n\ttrain_row = 0\n\ttest_row = 0\n\n\tdef decode_image(features):\n\t    # get BGR image from bytes\n\t    features[\"image\"] = cv2.imdecode(features[\"image\"], -1)\n\t    return features\n\t\n\t# Creating index files\n\tfor i in T(filelist):\n\t    tfrec_file = os.path.join(tfrec_dir, i)\n\t    index_file = os.path.join(tfrec_dir, i).replace('.tfrec', '.index')\n\t    if not os.path.exists(index_file):\n\t        os.system(f\"python3 -m tfrecord.tools.tfrecord2idx {tfrec_file} {index_file}\")\n\n\tdef tfrec_extract(filename):\n\t    global train_row\n\t    global test_row\n\t    global train\n\t    global test\n\t    tfrecord_path = os.path.join(tfrec_dir, filename)\n\t    index_path = tfrecord_path.replace('.tfrec', '.index')\n\t    \n\t    if 'train' in filename:\n\t        savedir = train_dir\n\t    else:savedir = test_dir\n\t    dataset = TFRecordDataset(tfrecord_path, index_path, transform=decode_image)\n\t    loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n\t    for data in T(loader):\n\t        if 'train' in filename:\n\t            train_row += 1\n\t        else: test_row += 1\n\t        img_name = data['image_name'].squeeze().data.cpu().numpy().copy()\n\t        img_name = os.path.join(savedir, ''.join(map(chr, img_name)))\n\t        img_name += '.jpg'\n\t        image_file = data['image'].squeeze().data.cpu().numpy()\n\t        cv2.imwrite(img_name, image_file)\n\t        del data['image']\n\t        del data['image_name']\n\t        for k, v in data.items():\n\t            if 'train' in filename:\n\t                train.loc[train_row, 'image_name'] = img_name\n\t                train.loc[train_row, k] = v.squeeze().data.cpu().numpy()\n\t                train.loc[train_row, 'tfrec'] = filename.replace('.tfrec', '')\n\t            else:\n\t                test.loc[test_row, 'image_name'] = img_name\n\t                test.loc[test_row, k] = v.squeeze().data.cpu().numpy()\n\t                test.loc[test_row, 'tfrec'] = filename.replace('.tfrec', '')\n\n\n\t# p_map(tfrec_extract, filelist)\n\tfor f in T(filelist):\n\t    tfrec_extract(f)\n\n\ttrain.to_csv(f\"{data_dir}/train_768.csv\", index=False)\n\ttest.to_csv(f\"{data_dir}/test_768.csv\", index=False)\n\tprint(f\"total {train_row} train images and {test_row} test images\")\n`",
    "945169": "Bhaia It would be better if you may share the the custom dataset that you created as \"TFRecordDataset \" .I am having error like \"no module named tfrecord\"",
    "945206": "My internet speed is very slow right now. :( I'll try to upload it once it gets stable. However, you can install `tfrecord` with this command: `pip3 install tfrecord`",
    "945281": "Okay bhai",
    "947841": "tawheedrony I've uploaded the dataset [here](www.kaggle.com/dataset/c1098e246be309c3f9781a78c372469a3bd7d51c2d963c30e1ebcd46d28bc70d). This dataset is created by combining [2020](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164092) and [2019-2017](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910) competition datasets provided by chris."
  },
  "source": "meta"
}