{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport PIL\nimport os\nimport random\nimport fastcore.all as fc\n\nfrom PIL import Image \nfrom pathlib import Path\nfrom tqdm import tqdm \nfrom functools import partial\nfrom concurrent.futures import ThreadPoolExecutor\n\n\nimport matplotlib.pyplot as plt\nplt.style.use(\"bmh\")\n%matplotlib inline ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-05T08:46:32.393896Z","iopub.execute_input":"2023-10-05T08:46:32.394285Z","iopub.status.idle":"2023-10-05T08:46:32.404577Z","shell.execute_reply.started":"2023-10-05T08:46:32.394257Z","shell.execute_reply":"2023-10-05T08:46:32.402179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Goal\nThe goal of this notebook is to create a tiny-imagenet dataset by compressing images to 32x32, so that we can pre-train a network on imagenet before fine-tuning on Fashion-MNIST.","metadata":{}},{"cell_type":"markdown","source":"## About ImageNet \n\n[source](https://image-net.org/download.php)\n\n> The dataset spans 1000 object classes and contains 1,281,167 training images, 50,000 validation images and 100,000 test images. ","metadata":{}},{"cell_type":"code","source":"root = \"/kaggle/input/imagenet-object-localization-challenge\"","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:35.095901Z","iopub.execute_input":"2023-10-05T08:17:35.096308Z","iopub.status.idle":"2023-10-05T08:17:35.101543Z","shell.execute_reply.started":"2023-10-05T08:17:35.096275Z","shell.execute_reply":"2023-10-05T08:17:35.100172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fc.L(fc.Path(root).glob(\"*.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:35.530483Z","iopub.execute_input":"2023-10-05T08:17:35.530901Z","iopub.status.idle":"2023-10-05T08:17:35.549316Z","shell.execute_reply.started":"2023-10-05T08:17:35.53087Z","shell.execute_reply":"2023-10-05T08:17:35.548519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_locs = pd.read_csv(Path(root)/\"LOC_train_solution.csv\")\nprint(img_locs.shape)\nimg_locs.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:36.058046Z","iopub.execute_input":"2023-10-05T08:17:36.058702Z","iopub.status.idle":"2023-10-05T08:17:37.388424Z","shell.execute_reply.started":"2023-10-05T08:17:36.058666Z","shell.execute_reply":"2023-10-05T08:17:37.387213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## lets count total of how many images are present ?","metadata":{}},{"cell_type":"code","source":"# images = fc.globtastic(fc.Path(root)/\"ILSVRC/Data/CLS-LOC\", file_glob=\"*.JPEG\", recursive=True)\n# images","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:37.391011Z","iopub.execute_input":"2023-10-05T08:17:37.392233Z","iopub.status.idle":"2023-10-05T08:17:37.39751Z","shell.execute_reply.started":"2023-10-05T08:17:37.392183Z","shell.execute_reply":"2023-10-05T08:17:37.395919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folders = fc.L((fc.Path(root)/\"ILSVRC/Data/CLS-LOC/train\").glob(\"*\"))\ntrain_folders","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:37.399371Z","iopub.execute_input":"2023-10-05T08:17:37.400214Z","iopub.status.idle":"2023-10-05T08:17:37.492444Z","shell.execute_reply.started":"2023-10-05T08:17:37.400149Z","shell.execute_reply":"2023-10-05T08:17:37.491245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fc.L(train_folders[0].glob(\"*.JPEG\"))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:17:37.495276Z","iopub.execute_input":"2023-10-05T08:17:37.496332Z","iopub.status.idle":"2023-10-05T08:17:37.697949Z","shell.execute_reply.started":"2023-10-05T08:17:37.496257Z","shell.execute_reply":"2023-10-05T08:17:37.696858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_imgs(folder):\n    return len(fc.L(folder.glob(\"*.JPEG\")))\ncount_imgs(train_folders[0])","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:43:03.595706Z","iopub.execute_input":"2023-10-05T08:43:03.596127Z","iopub.status.idle":"2023-10-05T08:43:03.614955Z","shell.execute_reply.started":"2023-10-05T08:43:03.596097Z","shell.execute_reply":"2023-10-05T08:43:03.613548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ThreadPoolExecutor(max_workers=12) as ex:\n    imgs_count = ex.map(count_imgs, train_folders)\nimgs_count = fc.L(imgs_count)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:43:14.301878Z","iopub.execute_input":"2023-10-05T08:43:14.302313Z","iopub.status.idle":"2023-10-05T08:43:43.766822Z","shell.execute_reply.started":"2023-10-05T08:43:14.302281Z","shell.execute_reply":"2023-10-05T08:43:43.76568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(4, 3))\nplt.hist(imgs_count)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:45:27.864175Z","iopub.execute_input":"2023-10-05T08:45:27.865367Z","iopub.status.idle":"2023-10-05T08:45:28.090577Z","shell.execute_reply.started":"2023-10-05T08:45:27.865328Z","shell.execute_reply":"2023-10-05T08:45:28.089153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(imgs_count), len(imgs_count)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:22:33.857692Z","iopub.execute_input":"2023-10-05T08:22:33.858712Z","iopub.status.idle":"2023-10-05T08:22:33.865616Z","shell.execute_reply.started":"2023-10-05T08:22:33.85867Z","shell.execute_reply":"2023-10-05T08:22:33.864848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_grid(imgs, rows, cols):\n    w,h = imgs[0].size\n    grid = Image.new('RGB', size=(cols*w, rows*h))\n    for i, img in enumerate(imgs): grid.paste(img, box=(i%cols*w, i//cols*h))\n    return grid","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:22:33.867114Z","iopub.execute_input":"2023-10-05T08:22:33.868171Z","iopub.status.idle":"2023-10-05T08:22:33.885897Z","shell.execute_reply.started":"2023-10-05T08:22:33.868137Z","shell.execute_reply":"2023-10-05T08:22:33.884869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thumbnail(img, size=256):\n    if not isinstance(img, PIL.Image.Image): img = Image.fromarray(img)\n    w, h = img.size\n    ar = h/w \n    return img.resize((size, int(size*ar)))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:24:16.398377Z","iopub.execute_input":"2023-10-05T08:24:16.398768Z","iopub.status.idle":"2023-10-05T08:24:16.404804Z","shell.execute_reply.started":"2023-10-05T08:24:16.398738Z","shell.execute_reply":"2023-10-05T08:24:16.403569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Visualize one class of Images. ","metadata":{}},{"cell_type":"code","source":"imgs = fc.L(train_folders[np.random.randint(len(train_folders))].glob(\"*.JPEG\"))\nrandom.shuffle(imgs)\nimage_grid([thumbnail(Image.open(i), 32) for i in imgs[:16]], rows=4, cols=4)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:24:38.020604Z","iopub.execute_input":"2023-10-05T08:24:38.020981Z","iopub.status.idle":"2023-10-05T08:24:43.058848Z","shell.execute_reply.started":"2023-10-05T08:24:38.020937Z","shell.execute_reply":"2023-10-05T08:24:43.057705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Single image size on disk","metadata":{}},{"cell_type":"code","source":"os.path.getsize(imgs[0])*1e-6","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:25:02.99658Z","iopub.execute_input":"2023-10-05T08:25:02.996944Z","iopub.status.idle":"2023-10-05T08:25:03.004622Z","shell.execute_reply.started":"2023-10-05T08:25:02.996916Z","shell.execute_reply":"2023-10-05T08:25:03.003815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> A folder size in MB","metadata":{}},{"cell_type":"code","source":"folder_size = [os.path.getsize(i)*1e-6 for i in imgs]\nsum(folder_size)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:25:06.958084Z","iopub.execute_input":"2023-10-05T08:25:06.958686Z","iopub.status.idle":"2023-10-05T08:25:13.468663Z","shell.execute_reply.started":"2023-10-05T08:25:06.958622Z","shell.execute_reply":"2023-10-05T08:25:13.467484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Get size of a path (folder/file) in Human readable format from terminal itself.","metadata":{}},{"cell_type":"code","source":"import subprocess\n\n#Copy from https://stackoverflow.com/questions/1392413/calculating-a-directorys-size-using-python\ndef du(path):\n    \"\"\"disk usage in human readable format (e.g. '2,1GB')\"\"\"\n    return subprocess.check_output(['du','-sh', path]).split()[0].decode('utf-8')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:25:13.470543Z","iopub.execute_input":"2023-10-05T08:25:13.471252Z","iopub.status.idle":"2023-10-05T08:25:13.476711Z","shell.execute_reply.started":"2023-10-05T08:25:13.471216Z","shell.execute_reply":"2023-10-05T08:25:13.475388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for num, d in enumerate(train_folders): \n    if num>10: break\n    size = du(d)\n    files_count = len(fc.L(d.glob(\"*.JPEG\")))\n    print(f\"{d.name}: total size: {size}: {files_count} files each with size:{int(1000*(int(size[:-1])/files_count))}KB\")","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:25:13.930079Z","iopub.execute_input":"2023-10-05T08:25:13.931012Z","iopub.status.idle":"2023-10-05T08:25:45.643799Z","shell.execute_reply.started":"2023-10-05T08:25:13.930944Z","shell.execute_reply":"2023-10-05T08:25:45.642601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dir = fc.Path(\"/kaggle/working/train/\")\n(save_dir).mkdir(exist_ok=True, parents=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:51.05323Z","iopub.execute_input":"2023-10-05T08:34:51.053588Z","iopub.status.idle":"2023-10-05T08:34:51.059397Z","shell.execute_reply.started":"2023-10-05T08:34:51.053559Z","shell.execute_reply":"2023-10-05T08:34:51.058147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folders[0].name","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:53.360069Z","iopub.execute_input":"2023-10-05T08:34:53.360438Z","iopub.status.idle":"2023-10-05T08:34:53.368131Z","shell.execute_reply.started":"2023-10-05T08:34:53.360412Z","shell.execute_reply":"2023-10-05T08:34:53.367034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(save_dir/train_folders[0].name).mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:53.807319Z","iopub.execute_input":"2023-10-05T08:34:53.807685Z","iopub.status.idle":"2023-10-05T08:34:53.813268Z","shell.execute_reply.started":"2023-10-05T08:34:53.807658Z","shell.execute_reply":"2023-10-05T08:34:53.812163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = fc.L(train_folders[0].glob(\"*.JPEG\"))\nimgs","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:54.908277Z","iopub.execute_input":"2023-10-05T08:34:54.908698Z","iopub.status.idle":"2023-10-05T08:34:54.922694Z","shell.execute_reply.started":"2023-10-05T08:34:54.908667Z","shell.execute_reply":"2023-10-05T08:34:54.921429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oimg = thumbnail(Image.open(imgs[0]), size=32)\noimg","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:56.109405Z","iopub.execute_input":"2023-10-05T08:34:56.110452Z","iopub.status.idle":"2023-10-05T08:34:56.125772Z","shell.execute_reply.started":"2023-10-05T08:34:56.110397Z","shell.execute_reply":"2023-10-05T08:34:56.124568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_loc = (save_dir/train_folders[0].name)/imgs[0].name\noimg.save(save_loc)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:34:57.177427Z","iopub.execute_input":"2023-10-05T08:34:57.178438Z","iopub.status.idle":"2023-10-05T08:34:57.184949Z","shell.execute_reply.started":"2023-10-05T08:34:57.178385Z","shell.execute_reply":"2023-10-05T08:34:57.184036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for num, d in enumerate(train_folders): \n    if num>3: break\n    files= fc.L(d.glob(\"*.JPEG\"))\n    save_folder = (save_dir/d.name)\n    save_folder.mkdir(exist_ok=True)\n    for file in files:\n        oimg = thumbnail(Image.open(file), size=32)\n        save_loc = (save_folder)/file.name\n        oimg.save(save_loc)\n    size = du(save_folder)\n    print(f\"{num}-{d.name}: {size}: {len(files)}: {int(1000*(float(size[:-1])/len(files)))}KB\")","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:35:05.583559Z","iopub.execute_input":"2023-10-05T08:35:05.583902Z","iopub.status.idle":"2023-10-05T08:37:48.075236Z","shell.execute_reply.started":"2023-10-05T08:35:05.583876Z","shell.execute_reply":"2023-10-05T08:37:48.073667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tiny_folder(folder, save_dir):\n    files= fc.L(folder.glob(\"*.JPEG\"))\n    save_folder = (save_dir/folder.name)\n    save_folder.mkdir(exist_ok=True)\n    for file in files:\n        oimg = thumbnail(Image.open(file), size=32)\n        save_loc = (save_folder)/file.name\n        oimg.save(save_loc)\n    size = du(save_folder)\n    print(f\"{num}-{folder.name}: {size}: {len(files)}: {int(1000*(float(size[:-1])/len(files)))}KB\")\n\ncreate_tiny_folder_parital = partial(create_tiny_folder, save_dir=save_dir)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:46:54.700047Z","iopub.execute_input":"2023-10-05T08:46:54.700924Z","iopub.status.idle":"2023-10-05T08:46:54.708682Z","shell.execute_reply.started":"2023-10-05T08:46:54.700891Z","shell.execute_reply":"2023-10-05T08:46:54.706894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ThreadPoolExecutor(max_workers=12) as ex:\n    results = ex.map(create_tiny_folder_parital, train_folders)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T08:47:17.662275Z","iopub.execute_input":"2023-10-05T08:47:17.662624Z","iopub.status.idle":"2023-10-05T08:48:23.1488Z","shell.execute_reply.started":"2023-10-05T08:47:17.662598Z","shell.execute_reply":"2023-10-05T08:48:23.147299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Done ","metadata":{}}]}