{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\n\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_CSV = '/kaggle/input/human-protein-atlas-train-val-split/train_df.csv'\n\nTRAIN_DIR = '/kaggle/input/human-protein-atlas-image-classification/train'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILTERS = ['red', 'green', 'blue', 'yellow']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(image_id, ddir):\n    \"\"\"\n    return: 4-channel PIL Image\n    \"\"\"\n    return Image.merge('RGBA', [Image.open(f\"{TRAIN_DIR}/{image_id}_{f}.png\") for f in FILTERS])\n\n\nclass ProteinLocalizationDataset(Dataset):\n    def __init__(self, df, ddir, transform=None):\n        self.df = df\n        self.ddir = ddir\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        image_id = self.df.loc[idx]['Id']\n        image = load_image(image_id, self.ddir)\n        if self.transform:\n            image = self.transform(image)\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = ProteinLocalizationDataset(train_df, TRAIN_DIR, transform=T.ToTensor())\ntrain_dl = DataLoader(train_ds, batch_size=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_stats(dl):\n    \n    # followed the calculations in http://notmatthancock.github.io/2017/03/23/simple-batch-stat-updates.html\n    \n    mean = 0\n    var = 0\n    size = 0\n\n    for imgs in dl:\n        # imgs.shape: (batch size, color channel, H, W)\n        b_mean = torch.mean(imgs, [0, 2, 3])\n        b_var = torch.var(imgs, [0, 2, 3])\n        b_size = imgs.size(0)\n\n        new_mean = size / (size + b_size) * mean \\\n                   + b_size / (size + b_size) * b_mean\n\n        new_var = size / (size + b_size) * var \\\n                  + b_size / (size + b_size) * b_var \\\n                  + (size * b_size) / (size + b_size) ** 2 * (mean - b_mean) ** 2\n\n        mean = new_mean\n        var = new_var\n        size += b_size\n    return (mean, torch.sqrt(var))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nstats = get_stats(train_dl)\nstats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(stats, 'stats.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.load('stats.pt')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}