{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Modifications by Stephen to handle all of the data and data splits"},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function, division\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.autograd import Variable\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms, utils\nimport matplotlib.pyplot as plt\nimport time\nimport copy\nimport os\n\n##added by Oskar\nfrom PIL import Image\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\nplt.ion()   # interactive mode\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Without use of Data loader:\n\nclass Scale(object):\n    \"\"\"Rescales the input PIL.Image to the given 'size'.\n    If 'size' is a 2-element tuple or list in the order of (width, height), it will be the exactly size to scale.\n    If 'size' is a number, it will indicate the size of the smaller edge.\n    For example, if height > width, then image will be\n    rescaled to (size * height / width, size)\n    size: size of the exactly size or the smaller edge\n    interpolation: Default: PIL.Image.BILINEAR\n    \"\"\"\n\n    def __init__(self, size, interpolation=Image.BILINEAR):\n        assert isinstance(size, int) or (isinstance(size, collections.Iterable) and len(size) == 2)\n        self.size = size\n        self.interpolation = interpolation\n\n    def __call__(self, img):\n        if isinstance(self.size, int):\n            w, h = img.size\n            if (w <= h and w == self.size) or (h <= w and h == self.size):\n                return img\n            if w < h:\n                ow = self.size\n                oh = int(self.size * h / w)\n                return img.resize((ow, oh), self.interpolation)\n            else:\n                oh = self.size\n                ow = int(self.size * w / h)\n                return img.resize((ow, oh), self.interpolation)\n        else:\n            return img.resize(self.size, self.interpolation)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split data, but only samples that are from the original dataset\n\nSPLIT_DATA = True\nif SPLIT_DATA:\n\n    import os\n    from shutil import copyfile\n\n    # Define parameters of the split\n    prop_train = 0.7\n    prop_val = 0.2\n    prop_test = 0.1\n    \n    def get_indices(num_samples, prop_train = 0.7, prop_dev = 0.2, prop_test = 0.1):\n    \n        assert (prop_train + prop_dev + prop_test - 1.0 < 1e-5)\n\n        num_train_samples = int(np.floor(num_samples * prop_train))\n        num_dev_samples = int(np.floor(num_samples * prop_dev))\n        num_test_samples = int(np.ceil(num_samples * prop_test))\n\n#         assert (num_train_samples + num_dev_samples + num_test_samples) == num_samples\n        \n        indices = list(range(num_samples))\n        np.random.seed(123)\n        np.random.shuffle(indices)\n\n        train_indices = indices[0:num_train_samples]\n        dev_indices = indices[num_train_samples:num_train_samples + num_dev_samples]\n        test_indices = indices[num_train_samples + num_dev_samples:]\n        return train_indices, dev_indices, test_indices\n\n    def maybe_makedir(dirname):\n        if not os.path.exists(dirname):\n            os.makedirs(dirname)\n\n    # Define directories\n    root_dir=\"../input/train\"\n    #root_dir = '../../data/all_data'\n#     train_path = root_dir + '/train'\n    train_path = root_dir\n    split_path = root_dir + '/split_new'\n\n    # Set the random seed\n    np.random.seed(456)\n\n    # Create split path if it does not exist\n    if not os.path.exists(split_path):\n        os.makedirs(split_path)\n\n    class_names = ('Type_1', 'Type_2', 'Type_3')\n    for class_name in class_names:\n\n        # Define the target location for the split class path\n        split_train_class_path = split_path + '/' + 'train' + '/' + class_name\n        split_val_class_path = split_path + '/' + 'val' + '/' + class_name\n        split_test_class_path = split_path + '/' + 'test' + '/' + class_name\n        \n        # Make directories if they dont already exist\n        maybe_makedir(split_train_class_path)\n        maybe_makedir(split_val_class_path)\n        maybe_makedir(split_test_class_path)\n\n        # Directory where original training data are stored\n        train_class_path = train_path + '/' + class_name\n\n        # Get list of files in train directory\n        train_files = os.listdir(train_class_path)\n        \n#         train_files_clean = filter(lambda k: 'additional' not in k, train_files)\n        train_files_clean = [k for k in train_files if 'additional' not in k]\n        train_files_additional = [k for k in train_files if 'additional' in k]\n#         print(train_files_clean)\n        \n        train_indices, val_indices, test_indices = get_indices(len(train_files_clean), prop_train, prop_val, prop_test)\n        print((len(train_indices), len(val_indices), len(test_indices), len(train_files)))\n\n        # Copy the train files to split directory\n        for index in train_indices:\n            src_path = train_class_path + '/' + train_files_clean[index]\n            dest_path = split_train_class_path + '/' + train_files_clean[index]\n            copyfile(src_path, dest_path)\n        # Copy all additional files to train directory\n        for additional_name in train_files_additional:\n            src_path = train_class_path + '/' + additional_name\n            dest_path = split_train_class_path + '/' + additional_name\n            copyfile(src_path, dest_path)\n        # Copy the val files to split directory\n        for index in val_indices:\n            src_path = train_class_path + '/' + train_files_clean[index]\n            dest_path = split_val_class_path + '/' + train_files_clean[index]\n            copyfile(src_path, dest_path)\n        # Copy the test files to split directory\n        for index in test_indices:\n            src_path = train_class_path + '/' + train_files_clean[index]\n            dest_path = split_test_class_path + '/' + train_files_clean[index]\n            copyfile(src_path, dest_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# test_folder = False\n# if test_folder:\n#     transform_scale = Scale(224)\n# else:\n#     transform_scale = Scale(256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#For splits that have already been defined\nhome = os.path.expanduser('~')\n\nload_path = home + '/data/all_data/split_new/'\n# load_path = home + '/data/train/'\nsave_path = home + '/data/all_data_scaled_alt/'\n\nclasses = ['Type_1/', 'Type_2/', 'Type_3/']\nsplits = ['train/', 'val/', 'test/']\n# splits = ['test/']\n\nfor split in splits:\n    # Get data split\n    if split == 'train/':\n        transform_scale = Scale(299)\n    else:\n        transform_scale = Scale(299)\n   \n    for c in classes:    \n        load_path_class = load_path + split + c\n        save_path_class = save_path + split + c\n        \n        # Make directories\n        if not os.path.exists(save_path + split):\n            os.makedirs(save_path + split)\n        if not os.path.exists(save_path_class):\n            os.makedirs(save_path_class)\n            \n        directory = os.fsencode(load_path_class)\n\n        for file in os.listdir(directory):\n            filename = os.fsdecode(file)\n            if filename.endswith(\".jpg\"): \n#                 print(load_path_class + filename)\n                im = Image.open(load_path_class + filename)\n                img = transform_scale(im)\n                img.save(save_path_class + filename)        \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}