{"cells":[{"metadata":{},"cell_type":"markdown","source":"#  Bercystan Central Committee Master Notebook","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Import Necessary Libraries**","execution_count":null},{"metadata":{"_uuid":"387d1788-f308-4fa5-aaea-4d15735a497f","_cell_guid":"5ae0a378-1a5b-4886-8281-3d812de4c8f0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport tensorflow as tf\nimport seaborn as sn\nimport matplotlib.pyplot as plt\nfrom keras import layers\nfrom tqdm import tqdm\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**View CWD Files**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input/siim-isic-melanoma-classification\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Read training and testing .csv files**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"training = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntesting = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntraining.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testing.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training.hist(column = \"age_approx\", bins = 10)\nplt.title(\"Training Ages\")\ntesting.hist(column = \"age_approx\", bins = 10)\nplt.title(\"Testing Ages\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sn.set(font_scale=1.2)\ntraining['sex'].value_counts().plot(kind='bar', figsize=(7, 6), rot=0)\nplt.title(\"Training Dataset by Gender\", y=1.02);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testing['sex'].value_counts().plot(kind='bar', figsize=(7, 6), rot=0)\nplt.title(\"Testing Dataset by Gender\", y=1.02);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"0 = Benign, 1 = Melanoma\")\ntraining['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Reading Images**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv2.imread(f\"../input/siim-isic-melanoma-classification/jpeg/train/ISIC_4131810.jpg\", cv2.IMREAD_UNCHANGED)\nprint(img.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Kevin here. So our jpeg images are RBG with 4k by 6k resolution which crashes memory if we load all of them. We either need to resize them, batch and save, or consider alternative file types (dcom etc). They give the exact same files using different file types. https://www.kaggle.com/parulpandey/melanoma-classification-eda-starter#2.-Reading-the-Image-datasets is a good place to start. For saving numpy arrays, this is the code I had for my own project:\n\nNote that I significantly resized my images to 150x150 (I only had 8 gigabytes of memory). Kaggle gives you 16, so you can afford a bit more resolution.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# read images from disk\ndef upload(img_size=(150, 150), dir=\"rawdata\"):\n    print(\"Loading Images...\")\n    landuses = [landuse for landuse in get_classes()]\n    for i in tqdm(range(len(landuses))):\n        for name in os.listdir(f\"files/{dir}/{landuses[i]}\"):\n            img = cv2.imread(\n                f\"files/{dir}/{landuses[i]}/{name}\", cv2.IMREAD_UNCHANGED)\n            img = cv2.resize(img, img_size)\n            yield img, landuses[i]\n\n\n# pickling images and labels to prevent re-uploading from rawdata every time\ndef serialize(name=\"Base\", dir=\"rawdata\", img_size=(150, 150)):\n    print(\"Serializing images...\")\n    images, labels = zip(*upload(img_size=img_size, dir=dir))\n    images = np.array(list(images))\n    labels = np.array(list(labels))\n\n    num_labels = []\n    current = labels[0]\n    index = 0\n    for label in labels:\n        if label != current:\n            index += 1\n            current = label\n        num_labels.append(index)\n    num_labels = np.array(num_labels)\n\n    with open(f\"files/{name}CompressedData.npz\", \"wb\") as file:\n        np.savez_compressed(file, images=images, labels=num_labels)\n\n\n# retrieve serialized images\ndef load(filename):\n    with open(f\"files/{filename}.npz\", \"rb\") as file:\n        arr = np.load(file)\n        return arr[\"images\"], arr[\"labels\"]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\n\nimage = Image.open(\"../input/siim-isic-melanoma-classification/jpeg/train/ISIC_4131810.jpg\")\nimage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def readImages(dataset=\"train\"):\n#     imgs = []\n#     count = 0\n#     for file in tqdm(os.listdir(f\"../input/siim-isic-melanoma-classification/jpeg/{dataset}\")):\n#         imgs.append(cv2.imread(f\"../input/siim-isic-melanoma-classification/jpeg/{dataset}/{file}\", cv2.IMREAD_UNCHANGED))\n#         print(file)\n#         count += 1\n#         if (count == 50):\n#             break\n#     return np.array(imgs)\n        \n# training_imgs = readImages(\"train\")\n# # testing_imgs = readImages(\"test\")\n# print(training_imgs.shape)\n# print(training_imgs[1])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}