{"cells":[{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"1f69e6b8-5f99-4cac-95e3-38389fbf237a","_cell_guid":"1445c960-a7e6-4740-940e-bafcde383d11","trusted":true},"cell_type":"code","source":"!pip install albumentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af89a040-95d5-49ba-bb59-eed78dedff0a","_cell_guid":"7faae6ad-2c75-4b64-aa3f-fae0807a1aee","trusted":true},"cell_type":"code","source":"import torch\nimport albumentations\nfrom albumentations import ( Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, IAAAdditiveGaussianNoise, Transpose, ToGray )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport matplotlib.pyplot as plt\n\nseed = 42\n\nimport pandas as pd\nimport os\nfrom torch.utils.data import Dataset,DataLoader\nfrom tqdm import tqdm\n\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53d58a4e-bbf9-4c89-9311-475775b4aa2c","_cell_guid":"6add45fe-34fb-41f1-a98a-90bfcd737f25","trusted":true},"cell_type":"code","source":"class Ranzcr_jpg_train_dataset(Dataset):    \n\tdef __init__(self, files_folder_path, df, num_channels , transfroms = None ):\n\t\tself.files_folder_path = files_folder_path\n\t\tself.df = df\n\t\tself.transforms = transfroms\n\t\tself.num_channels = num_channels\n\n\tdef __len__(self):\n\t\treturn len(self.df)\n\n\tdef __getitem__(self, idx):\n\n\t\timage_id = self.df.StudyInstanceUID.values[idx]\n        \n\t\tif self.num_channels == 1:\n\t\t\timage = cv2.imread(os.path.join(self.files_folder_path, image_id + \".jpg\" ), 0)\n\n\t\telse:\n\t\t\timage = cv2.imread(os.path.join(self.files_folder_path, image_id + \".jpg\" ))\n\t\t\timage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\t\t\n\n\t\tif self.transforms:\n\t\t\timage = self.transforms(image=image)['image']\n\n\t\tlabels = self.df[self.df.StudyInstanceUID == image_id].values.tolist()[0][1:-1]\n\t\tlabels = torch.tensor(labels,dtype= torch.float32) #.view(1,-1)\n\n\t\treturn image, labels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a98a696-c072-4ebb-ab18-89f167062a99","_cell_guid":"0a640f66-daf1-4038-b127-4d37c172a369","trusted":true},"cell_type":"code","source":"train_path = '../input/ranzcr-clip-catheter-line-classification/train'\ntrain_files = os.listdir(train_path)\n\ntrain_df = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\n#train_df = pd.read_csv('../input/ranzcr/X_test_small_250_seed_42.csv')\n\ntrain = train_df.reset_index(drop=True) # reset index on both dataframes\n\nprint(train.shape)\n\nnum_channel = 3\n\ndata_mean = [0.485,0.456,0.406]\ndata_std = [0.229,0.224,0.225]\n\n#Image augmentation\nimg_size = 255\nprint(data_mean , data_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef Show_Xrays(augmentation):\n    num_channels = 3\n    trainset = Ranzcr_jpg_train_dataset(train_path,  train, num_channels , augmentation )\n    trainloader = DataLoader(trainset, batch_size = 4 , num_workers = 0 , shuffle = False)\n\n    fig = plt.figure()\n    fig.set_size_inches(15, 15)\n    #fig.savefig('test2png.png', dpi=100)\n\n    for batch_i, (data, target) in tqdm(enumerate(trainloader)):\n        #print(data.shape)\n        if batch_i == 1:\n            break\n        for i in range(data.shape[0]):\n            ax = plt.subplot(2 ,2, i+1)\n            plt.tight_layout()\n            ax.axis('off')\n            plt.imshow(data[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Baseline"},{"metadata":{"_uuid":"65e1ad07-1a45-47ef-a60e-9f0d98ee57b1","_cell_guid":"ca8a9a5c-2b25-4280-897b-ab1c227b06bd","trusted":true},"cell_type":"code","source":"train_augs = albumentations.Compose([ albumentations.Resize(height=img_size, width=img_size, p=1.0), albumentations.Normalize(mean= data_mean,std= data_std ,),])\n\nShow_Xrays(train_augs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_augs = albumentations.Compose([   albumentations.Resize(img_size, img_size),    ToTensorV2()     ])\n\nnum_channels = 3\ntrainset = Ranzcr_jpg_train_dataset(train_path,  train, num_channels , train_augs )\ntrainloader = DataLoader(trainset, batch_size = 64 , num_workers = 1 , shuffle = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for batch_i, (data, target) in tqdm(enumerate(trainloader)):\n        print(data.shape)\n        if batch_i == 1:\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[0,:, 1,1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://github.com/aladdinpersson/Machine-Learning-Collection/blob/master/ML/Pytorch/Basics/pytorch_std_mean.py\nhttps://www.youtube.com/watch?v=y6IEcEBRZks"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_augs = albumentations.Compose([   albumentations.Resize(img_size, img_size),    ToTensorV2()     ])\n\nnum_channels = 3\ntrainset = Ranzcr_jpg_train_dataset(train_path,  train, num_channels , train_augs )\ntrainloader = DataLoader(trainset, batch_size = 64 , num_workers = 1 , shuffle = True)\n\ndef get_mean_std(loader):\n    # var[X] = E[X**2] - E[X]**2\n    channels_sum, channels_sqrd_sum, num_batches = 0, 0, 0\n    for data, _ in tqdm(loader):\n        channels_sum += torch.Tensor.float(data ).mean(dim=[0, 2, 3])\n        print(num_batches +1  , channels_sum/num_batches+1)\n        channels_sqrd_sum += torch.Tensor.float(data ** 2).mean(dim=[0, 2, 3])\n        print(num_batches +1  , channels_sqrd_sum/num_batches+1)\n        num_batches += 1\n    \n    mean = channels_sum / num_batches\n    std = (channels_sqrd_sum / num_batches - mean ** 2) ** 0.5\n    std_not_minus = (channels_sqrd_sum / num_batches ) ** 0.5\n    \n    return mean, std , std_not_minus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean, std , mean_std = get_mean_std(trainloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean, std , mean_std = get_mean_std(trainloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = [124.3329, 124.3329, 124.3329]\nstd = [103.0483, 103.0483, 103.0483]\n\nstd = (((std - mean) ** 2) * (1 / num_batches))  ** 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = [123.0365, 123.0365, 123.0365]\n#mean = mean*(1/255)\nstd = [0.5,0.5,0.5]\n\ndata_mean = [0.485,0.456,0.406]\ndata_std = [0.229,0.224,0.225\n            \nmean = [130.4711, 130.4711, 130.4711]\nmean = mean*(1/255)\nprint(mean)\nstd = [108.1780, 108.1780, 108.1780]\nstd = std*(1/255)\nprint(std)\n            \n            \n57 tensor([126.3339, 126.3339, 126.3339])\n57 tensor([104.8026, 104.8026, 104.8026])\n\n92 tensor([125.4034, 125.4034, 125.4034])\n92 tensor([104.0455, 104.0455, 104.0455])\n\n            \n114 tensor([125.3113, 125.3113, 125.3113])\n114 tensor([103.8559, 103.8559, 103.8559])\n\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"123.0365/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_augs = albumentations.Compose([ albumentations.Resize(height=img_size, width=img_size, p=1.0), \n                                      albumentations.Normalize(mean= mean, std= std ),])\n\nShow_Xrays(train_augs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"R_channel = 0\nG_channel = 0\nB_channel = 0\n\npathDir = '../input/ranzcr-clip-catheter-line-classification/test/*.jpg'\n\nfrom glob import glob\nimg_list = glob(pathDir)\nprint(len(img_list))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"R_total , G_total ,B_total = 0, 0 , 0\n\ntotal_pixel = 0\nfor idx in range(len(img_list)):\n    filename = img_list[idx]\n    img = plt.imread(filename)\n\n    total_pixel = total_pixel + img.shape[0] * img.shape[1]\n\n    R_total += np.sum((img[:, :, 0] - R_mean) ** 2)\n    G_total = G_total + np.sum((img[:, :, 1] - G_mean) ** 2)\n    B_total = B_total + np.sum((img[:, :, 2] - B_mean) ** 2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nR_std = sqrt(R_total / total_count)\nG_std = sqrt(G_total / total_count)\nB_std = sqrt(B_total / total_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindata = datasets.ImageFolder('../input/ranzcr-clip-catheter-line-classification/test/', transforms.ToTensor())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_means = torch.stack([t.mean(1).mean(1) for t, c in traindata])\nimage_means.mean(0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}