{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 合并数据的路径"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\nTRAIN_DIR = './input/cassavapreprocessed/train_images/train_images'\nTRAIN_CSV_PATH = '../input/cassavapreprocessed/merged_data.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_CSV_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train['source'] = train['image_id'].apply(lambda x:2019 if str(x)[:5]=='train' else 2020)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 计算均值与标准差"},{"metadata":{"trusted":true},"cell_type":"code","source":"# coding:utf-8\nimport os\nimport numpy as np\nfrom torchvision.datasets import ImageFolder\nimport torchvision.transforms as transforms\n\nimport pickle\n\n\"\"\"\n    在训练前先运行该函数获得数据的均值和标准差\n\"\"\"\n\nclass Dataloader():\n    def __init__(self, dataroot):\n        # 训练，验证，测试数据集文件夹名\n        # self.dataroot = dataroot\n        self.dirs = ['train_images']\n\n        self.means = [0, 0, 0]        \n        self.stdevs = [0, 0, 0]\n\n        self.transform = transforms.Compose([transforms.Resize((512,512)),\n                                             transforms.ToTensor(),  # 数据值从[0,255]范围转为[0,1]，相当于除以255操作\n                                             # transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225))\n                                             ])\n\n        # 因为这里使用的是ImageFolder，按文件夹给数据分类，一个文件夹为一类，label会自动标注好\n        self.dataset = {x: ImageFolder(os.path.join(dataroot, x), self.transform) for x in self.dirs}\n        \n        \n        \n\n    def get_mean_std(self):\n        \"\"\"\n        计算数据集的均值和标准差\n        :param type: 使用的是那个数据集的数据，有'train', 'test', 'testing'\n        :param mean_std_path: 计算出来的均值和标准差存储的文件\n        :return:\n        \"\"\"\n        num_imgs = len(self.dataset['train_images'])\n        # print(num_imgs)\n        for data in self.dataset['train_images']:\n            \n            img = data[0]            \n            for i in range(3):\n                # 一个通道的均值和标准差\n                self.means[i] += img[i, :, :].mean()\n                self.stdevs[i] += img[i, :, :].std()\n               \n        self.means = np.asarray(self.means) / num_imgs\n        self.stdevs = np.asarray(self.stdevs) / num_imgs\n\n        print(\"{} : normMean = {}\".format(type, self.means))\n        print(\"{} : normstdevs = {}\".format(type, self.stdevs))\n\n# # 将得到的均值和标准差写到文件中，之后就能够从中读取\n# with open(mean_std_path, 'wb') as f:\n# pickle.dump(self.means, f)\n# pickle.dump(self.stdevs, f)\n# print('pickle done')\n\nif __name__ == '__main__':\n\n    dataroot = '../input/cassavapreprocessed/'\n    dataloader = Dataloader(dataroot)\n    # for x in dataloader.dirs:\n    # mean_std_path = 'mean_std_value_' + x + '.pkl'\n    dataloader.get_mean_std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}