{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install albumentations\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom torchvision import datasets, transforms, models\nimport os\nimport shutil\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch.nn.functional as F\nimport torch.nn as nn\n#torchvision.utils.save_image\nfrom torchvision import datasets, transforms ,utils\nprint(os.listdir(\"../input\"))\nfrom PIL import Image\nimport numpy as np\nimport cv2\nfrom matplotlib import pyplot as plt\nfrom IPython.display import display, HTML \nfrom matplotlib.pyplot import imshow\nimport numpy as np\nfrom PIL import Image\nimport math\nimport os\nfrom albumentations.augmentations.transforms import *\nfrom albumentations import (\n Compose, ToFloat, FromFloat, RandomRotate90, Flip, OneOf, MotionBlur, MedianBlur, Blur,\n ShiftScaleRotate, OpticalDistortion, GridDistortion, RandomBrightnessContrast,\n HueSaturationValue,\n)\n##!pip install pretrainedmodels\n#import pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsample = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\ntrain = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nfrom sklearn.model_selection import train_test_split\n#train_80 = train.sample(frac=0.8)\ntrain_80,validate = train_test_split(train,stratify=train['diagnosis'],test_size=0.2)\ntrain_80['image_loc']='/kaggle/images/train/'\ntrain_80['basedir']='/kaggle/input/aptos2019-blindness-detection/train_images/'\nvalidate = train.loc[~train.index.isin(train_80.index)]\nvalidate['image_loc']='/kaggle/images/val/'\nvalidate['basedir']='/kaggle/input/aptos2019-blindness-detection/train_images/'\ntrain.shape,train_80.shape,validate.shape,test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([train_80,validate])\ntrain = train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_80['diagnosis'].value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validate['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augment(aug, image):\n return aug(image=image)['image']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n if img.ndim ==2:\n mask = img>tol\n return img[np.ix_(mask.any(1),mask.any(0))]\n elif img.ndim==3:\n gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n mask = gray_img>tol\n \n check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n if (check_shape == 0): # image is too dark so that we crop out everything,\n return img # return original image\n else:\n img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n # print(img1.shape,img2.shape,img3.shape)\n img = np.stack([img1,img2,img3],axis=-1)\n # print(img.shape)\n return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_image(p=0.5):\n return Compose([\n #ToFloat(),\n OneOf([RandomRotate90(),\n Flip()]),\n #RandomRotate90(),\n #Flip(),\n #RandomSizedBBoxSafeCrop(224, 224, erosion_rate=0.0, interpolation=1, always_apply=False, p=1.0),\n #MedianBlur(blur_limit=3, p=1.0),\n #OneOf([\n # MotionBlur(p=0.2),\n # GaussianBlur(blur_limit=5, always_apply=True, p=0.5),\n # MedianBlur(blur_limit=3, p=0.1),\n # Blur(blur_limit=3, p=0.1),\n #], p=0.2),\n #CLAHE(clip_limit=(2,2), tile_grid_size=(10, 10), always_apply=True, p=0.5),\n #InvertImg(always_apply=True, p=1.0),\n #GaussianBlur(blur_limit=5, always_apply=True, p=1.0),\n #RGBShift(r_shift_limit=(20,20), g_shift_limit=(30,30), b_shift_limit=(40,40), always_apply=True, p=1.0),\n #ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=1.0),\n #RandomBrightnessContrast(brightness_limit=(-0.2,0.4), contrast_limit=(0.2,0.2), always_apply=False, p=0.5),\n #GaussianBlur(blur_limit=7, always_apply=True, p=1.0),\n #MedianBlur(blur_limit=3, p=1.0),\n #Blur(blur_limit=3, p=1.0),\n #OpticalDistortion(p=1.0),\n #OneOf([\n # OpticalDistortion(p=0.3),\n # GridDistortion(p=0.1),\n #], p=0.2), \n #HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, always_apply=False, p=0.5),\n #HueSaturationValue(hue_shift_limit=(-2,2), sat_shift_limit=20, val_shift_limit=20, p=1.0),\n ], p=p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def al_transform(imageloc):\n img = cv2.imread(imageloc)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n #img = img*2\n #img = crop_image_from_gray(img,tol=7)\n #augmentation = transform_image(p=0.9)\n #final = augment(augmentation,img)\n #final = augment(augmentation,np.array(img))\n #final = final*2\n #print(final)\n #print(augmentation)\n #img = augmentation(image=img)['image']\n #final = Image.fromarray(img)\n return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basedir = '/kaggle/input/aptos2019-blindness-detection/train_images/6194e0fff071.png'\n#img = cv2.imread(basedir)\nimg = al_transform(basedir)\n#newimagepath = newdirname + '/'+row['id_code'] +'.png'\n#cv2.imwrite(newimagepath, image)\nplt.figure(figsize=(6, 6))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#basedir = '../input/aptos2019-blindness-detection/train_images/61bbc11fe503.png'\nimg = cv2.imread(basedir)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n#img = al_transform(basedir)\n#newimagepath = newdirname + '/'+row['id_code'] +'.png'\n#cv2.imwrite(newimagepath, image)\nplt.figure(figsize=(6, 6))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!rm -rf '../imagestrain'\n#!rm -rf '../imagestest'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainfilenames = train['image'].tolist()\n#basedir = '../input/aptos2019-blindness-detection/train_images/'\n#destinationfolder = '/kaggle/images/'\nfor i,row in train.iterrows():\n currentfileloc = str(row['basedir']) + row['id_code'] +'.png'\n newdirname = str(row['image_loc']) +str(row['diagnosis'])\n #print(currentfileloc)\n if not os.path.exists(newdirname):\n os.makedirs(newdirname)\n image = al_transform(str(currentfileloc))\n newimagepath = newdirname + '/'+row['id_code'] +'.png'\n cv2.imwrite(newimagepath, image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basedir = '/kaggle/input/aptos2019-blindness-detection/test_images/'\ndestinationfolder = '/kaggle/images/'\nfor i,row in test.iterrows():\n currentfileloc = basedir + row['id_code'] +'.png'\n newdirname = destinationfolder + 'test/'\n if not os.path.exists(newdirname):\n os.makedirs(newdirname)\n image = al_transform(currentfileloc)\n newimagepath = newdirname + row['id_code'] +'.png'\n cv2.imwrite(newimagepath, image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/images/train/0 | wc -l\n!ls /kaggle/images/train/1 | wc -l\n!ls /kaggle/images/train/2 | wc -l\n!ls /kaggle/images/train/3 | wc -l\n!ls /kaggle/images/train/4 | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_images(dirname,diagnosis,newdirname,no_of_images):\n files = [dirname+file+'.png' for file in train[train.diagnosis==diagnosis]['id_code'].tolist()]\n if no_of_images > 0:\n files = random.sample(files, no_of_images)\n for file in files:\n image = al_transform(file)\n newimagepath = newdirname + str(diagnosis) +'/'+file.split('/')[-1].split('.')[0] + '_enh'+'.png'\n #print(newimagepath)\n cv2.imwrite(newimagepath, image)\ndirname = '../input/aptos2019-blindness-detection/train_images/' \nnewdirname = '../images/train/'\nimagenum=[0,0,0,0]\ndiagnosis = [1,2,3,4]\n#for imagenum,diagnosis in zip(imagenum,diagnosis):\n# new_images(dirname,diagnosis,newdirname,imagenum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/images/train/0 | wc -l\n!ls /kaggle/images/train/1 | wc -l\n!ls /kaggle/images/train/2 | wc -l\n!ls /kaggle/images/train/3 | wc -l\n!ls /kaggle/images/train/4 | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/images/val/0 | wc -l\n!ls /kaggle/images/val/1 | wc -l\n!ls /kaggle/images/val/2 | wc -l\n!ls /kaggle/images/val/3 | wc -l\n!ls /kaggle/images/val/4 | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basedir = '../images/test/0005cfc8afb6.png'\nimg = cv2.imread(basedir)\nimg = al_transform(currentfileloc)\n#newimagepath = newdirname + '/'+row['id_code'] +'.png'\n#cv2.imwrite(newimagepath, image)\nplt.figure(figsize=(6, 6))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../images/test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basedir = '../input/aptos2019-blindness-detection/train_images/042470a92154.png'\n#img = cv2.imread(basedir)\nimg = al_transform(basedir)\n#newimagepath = newdirname + '/'+row['id_code'] +'.png'\n#cv2.imwrite(newimagepath, image)\nplt.figure(figsize=(10, 10))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"convertlabeldict = {0: 'No DR', \n1:'Mild', \n2:'Moderate', \n3:'Severe', \n4:'Proliferative DR'}\n#train['diagnosis'] = train['diagnosis'].map(convertlabeldict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../imagestrain/3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = '../imagestrain/3/50a2aef380c8.png'\npil_im = Image.open(data, 'r')\nimshow(np.asarray(pil_im))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_weights_for_balanced_classes(images, nclasses): \n count = [0] * nclasses \n for item in images: \n count[item[1]] += 1 \n weight_per_class = [0.] * nclasses \n N = float(sum(count)) \n for i in range(nclasses): \n weight_per_class[i] = N/float(count[i]) \n weight = [0] * len(images) \n for idx, val in enumerate(images): \n weight[idx] = weight_per_class[val[1]] \n return weight \n\ndef load_split_train_test(datadir, batch_size=32,image_size=224,train_ind=True):\n train_transforms = transforms.Compose([transforms.Resize(256),\n transforms.RandomHorizontalFlip(),\n transforms.RandomRotation(10),\n transforms.CenterCrop(image_size),\n transforms.ToTensor(),\n transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n ])\n data_set = datasets.ImageFolder(datadir,transform=train_transforms)\n \n if train_ind:\n weights = make_weights_for_balanced_classes(data_set.imgs, len(data_set.classes)) \n weights = torch.DoubleTensor(weights) \n sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights)) \n loader = torch.utils.data.DataLoader(data_set, batch_size=batch_size, #shuffle = True, \n sampler = sampler, \n num_workers=4, pin_memory=True)\n else:\n sampler=None\n loader = torch.utils.data.DataLoader(data_set, batch_size=batch_size, shuffle = True, \n sampler = sampler, \n num_workers=4, pin_memory=True)\n\n return loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../images/train'\nbatch_size = 32\nimage_size=224\n#valsize = 0.2\n#train_transforms = transforms.Compose([transforms.Resize(image_size),\n# transforms.ToTensor(),\n# transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n# ])\n#training_loader = load_split_train_test(data_dir,batch_size,image_size,train_ind=True)\n#train_data = datasets.ImageFolder(data_dir,transform=train_transforms)\n#trainloader = torch.utils.data.DataLoader(train_data,batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for data,target in training_loader:\n# print(data)\n# print(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = ('No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative DR') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checkpoint_densenet201 = torch.load(\"../input/pytorch-pretrained-image-models/densenet201.pth\")\n#checkpoint_densenet121 = torch.load(\"../input/pytorch-pretrained-image-models/densenet121.pth\")\n#checkpoint_resnet50 = torch.load(\"../input/pytorch-pretrained-image-models/resnet50.pth\")\n#checkpoint_resnet34 = torch.load(\"../input/pytorch-pretrained-image-models/resnet34.pth\")\n#checkpoint_inceptionv3 = torch.load(\"../input/pretrained-pytorch-models/inception_v3_google-1a9a5a14.pth\")\n#checkpoint_densenet161 = torch.load(\"../input/pretrained-pytorch-models/densenet161-17b70270.pth\")\n#checkpoint_densenet161 = torch.load(\"../input/resnet152/resnet152.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def definemodel(modelnumber,pretrained=False,freezelonlylastlayer = 'yes',lr=0.0001):\n train_data_dir = '/kaggle/images/train'\n validate_data_dir = '/kaggle/images/val'\n valsize=0.2\n if modelnumber == 0:\n batch_size = 32\n image_size=224\n #training_loader, validation_loader = load_split_train_test(data_dir,batch_size,valsize,image_size)\n training_loader = load_split_train_test(train_data_dir,batch_size,image_size,train_ind=False)\n validation_loader = load_split_train_test(validate_data_dir,batch_size,image_size,train_ind=False)\n if pretrained:\n model = models.densenet201(pretrained=True)\n else:\n model = models.densenet201(pretrained=False)\n checkpoint = torch.load(\"../input/pytorch-pretrained-image-models/densenet201.pth\")\n model.load_state_dict(checkpoint)\n #criterion = nn.CrossEntropyLoss() \n if freezelonlylastlayer == 'yes': \n for param in model.parameters():\n param.requires_grad = False\n model.classifier = nn.Sequential( \n nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=1920, out_features=1920, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=1920, out_features=5, bias=True),)\n optimizer = torch.optim.Adam(model.classifier.parameters(), lr = lr, weight_decay=1e-5) \n else:\n model.classifier = nn.Sequential( \n nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=1920, out_features=1920, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=1920, out_features=5, bias=True),)\n optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=1e-5) \n criterion = nn.CrossEntropyLoss()\n modelname = 'densenet201'\n elif modelnumber == 1:\n batch_size = 64\n image_size=224\n training_loader = load_split_train_test(train_data_dir,batch_size,image_size,train_ind=False)\n validation_loader = load_split_train_test(validate_data_dir,batch_size,image_size,train_ind=False)\n #training_loader, validation_loader = load_split_train_test(data_dir,batch_size,valsize,image_size)\n #model = models.resnet50(pretrained=True)\n if pretrained:\n model = models.resnet50(pretrained=True)\n else:\n model = models.resnet50(pretrained=False)\n checkpoint = torch.load(\"../input/pytorch-pretrained-image-models/resnet50.pth\")\n model.load_state_dict(checkpoint)\n #for param in model2.parameters():\n # param.requires_grad = False\n\n #criterion = nn.CrossEntropyLoss()\n \n if freezelonlylastlayer == 'yes': \n for param in model.parameters():\n param.requires_grad = False\n model.fc = nn.Sequential( \n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),)\n optimizer = torch.optim.Adam(model.fc.parameters(), lr = lr, weight_decay=1e-5) \n else:\n model.fc = nn.Sequential( \n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),)\n optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=1e-5) \n modelname = 'resnet50'\n criterion = nn.CrossEntropyLoss()\n elif modelnumber == 2:\n batch_size = 32\n image_size=299\n #training_loader, validation_loader = load_split_train_test(data_dir,batch_size,valsize,image_size)\n training_loader = load_split_train_test(train_data_dir,batch_size,image_size,train_ind=False)\n validation_loader = load_split_train_test(validate_data_dir,batch_size,image_size,train_ind=False)\n #model = models.inception_v3(pretrained=True)\n if pretrained:\n model = models.inception_v3(pretrained=True)\n else:\n model = models.inception_v3(pretrained=False)\n checkpoint = torch.load(\"../input/pretrained-pytorch-models/inception_v3_google-1a9a5a14.pth\")\n model.load_state_dict(checkpoint)\n #for param in model5.parameters():\n # param.requires_grad = False\n #criterion = nn.CrossEntropyLoss() \n if freezelonlylastlayer == 'yes': \n for param in model.parameters():\n param.requires_grad = False\n model.fc = nn.Sequential(\n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),\n )\n\n optimizer = torch.optim.Adam(model.fc.parameters(), lr = lr, weight_decay=1e-5) \n else:\n model.fc = nn.Sequential(\n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),\n )\n optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=1e-5) \n modelname = 'inception'\n criterion = nn.CrossEntropyLoss()\n elif modelnumber == 3:\n batch_size = 32\n image_size=224\n training_loader = load_split_train_test(train_data_dir,batch_size,image_size,train_ind=False)\n validation_loader = load_split_train_test(validate_data_dir,batch_size,image_size,train_ind=False)\n #training_loader, validation_loader = load_split_train_test(data_dir,batch_size,valsize,image_size)\n #model = models.densenet161(pretrained=True)\n if pretrained:\n model = models.densenet161(pretrained=True)\n else:\n model = models.densenet161(pretrained=False)\n checkpoint = torch.load(\"../input/pytorch-model-zoo/densenet161-347e6b360.pth\")\n model.load_state_dict(checkpoint)\n #model = models.resnet152(pretrained=False)\n #checkpoint = torch.load(\"../input/resnet152/resnet152.pth\")\n #model.load_state_dict(checkpoint)\n #for param in model5.parameters():\n # param.requires_grad = False\n #criterion = nn.CrossEntropyLoss() \n if freezelonlylastlayer == 'yes': \n for param in model.parameters():\n param.requires_grad = False\n model.classifier = nn.Sequential(\n nn.BatchNorm1d(2208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.4),\n nn.Linear(in_features=2208, out_features=2208, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2208, out_features=5, bias=True),\n )\n optimizer = torch.optim.Adam(model.fc.parameters(), lr = lr, weight_decay=1e-5) \n else:\n model.classifier = nn.Sequential( \n nn.BatchNorm1d(2208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.25),\n nn.Linear(in_features=2208, out_features=2208, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2208, out_features=5, bias=True),\n )\n optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=1e-5) \n modelname = 'densenet161'\n criterion = nn.CrossEntropyLoss()\n #model.to(device)\n #model3.to(device)\n elif modelnumber == 4:\n batch_size = 64\n image_size=224\n training_loader = load_split_train_test(train_data_dir,batch_size,image_size,train_ind=False)\n validation_loader = load_split_train_test(validate_data_dir,batch_size,image_size,train_ind=False)\n #training_loader, validation_loader = load_split_train_test(data_dir,batch_size,valsize,image_size)\n #print(len(training_loader))\n #\n #print(len(validation_loader))\n #model = models.resnet152(pretrained=True)\n if pretrained:\n model = models.resnet152(pretrained=True)\n else:\n model = models.resnet152(pretrained=False)\n checkpoint = torch.load(\"../input/resnet152/resnet152.pth\")\n model.load_state_dict(checkpoint)\n #for param in model2.parameters():\n # param.requires_grad = False\n\n #criterion = nn.CrossEntropyLoss()\n \n if freezelonlylastlayer == 'yes': \n for param in model.parameters():\n param.requires_grad = False\n model.fc = nn.Sequential( \n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.25),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),\n )\n optimizer = torch.optim.Adam(model.fc.parameters(), lr = lr, weight_decay=1e-5) \n else:\n model.fc = nn.Sequential( \n nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n nn.Dropout(p=0.25),\n nn.Linear(in_features=2048, out_features=2048, bias=True),\n nn.LeakyReLU(),\n #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n #nn.Dropout(p=0.5),\n nn.Linear(in_features=2048, out_features=5, bias=True),\n )\n optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=1e-5) \n modelname = 'resnet152'\n criterion = nn.CrossEntropyLoss()\n return model,criterion,optimizer,modelname,training_loader, validation_loader,image_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\ndef quadratic_kappa(y_hat, y):\n return torch.tensor(cohen_kappa_score(torch.round(y_hat.type(torch.FloatTensor)), y.type(torch.FloatTensor), weights='quadratic'),device='cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cyclical_lr(stepsize, min_lr=3e-4, max_lr=3e-3):\n\n # Scaler: we can adapt this if we do not want the triangular CLR\n scaler = lambda x: 1.\n\n # Lambda function to calculate the LR\n lr_lambda = lambda it: min_lr + (max_lr - min_lr) * relative(it, stepsize)\n \n # Additional function to see where on the cycle we are\n def relative(it, stepsize):\n cycle = math.floor(1 + it / (2 * stepsize))\n x = abs(it / stepsize - 2 * cycle + 1)\n #print(max(0, (1 - x)) * scaler(cycle))\n return max(0, (1 - x)) * scaler(cycle)\n\n return lr_lambda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def modeltrainv2(model,criterion,optimizer,training_loader,validation_loader,epochs = 10,\n modeltype = 'other',freezer=[0,0,0,0,0,0,0,0,0,0],stepsize=4,factor=2\n ,end_lr=0.001,sch_lr='sgd_cyclic'):\n running_loss_history = []\n running_corrects_history = []\n val_running_loss_history = []\n val_running_corrects_history = []\n step_size = stepsize*len(training_loader)\n print(f\"Step Size id {step_size}\")\n if sch_lr == 'cyclic':\n clr = cyclical_lr(step_size, min_lr=end_lr/factor, max_lr=end_lr)\n scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, [clr])\n ###elif sch_lr == 'sgd_cyclic' :\n #optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n # scheduler = torch.optim.CyclicLR(optimizer)\n elif sch_lr == 'cosine' :\n scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0, last_epoch=-1)\n elif sch_lr == 'zero' :\n pass\n elif sch_lr == 'red_lr_p':\n scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')\n elif sch_lr=='cyclic_lr':\n scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.0001, max_lr=0.005)\n #batch = 0\n #lr_lambda = lambda x: math.exp(x * math.log(end_lr / start_lr) / (lr_find_epochs * len( dataloaders[\"train\"])))\n #scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n layers=[]\n# for child in model.children():\n# layers.append(child)\n for child in model.children():\n for c in child.children():\n# for c1 in c.children():\n layers.append(c)\n no_of_layers = len(layers)\n print(f\"Number of layers in the model is {no_of_layers}\")\n for e in range(epochs):\n batch = 0\n running_loss = 0.0\n running_corrects = 0.0\n val_running_loss = 0.0\n val_running_corrects = 0.0\n kappa_score=0.0\n for child in model.children():\n for c in child.children():\n for param in c.parameters():\n param.requires_grad = True\n layerf = freezer[e]\n print(f\"Input freezed layers are {layerf}\")\n cntr=0\n freezelayer=0\n if no_of_layers <= freezer[e]:\n print(f\"freezer mentioned is {freezer[e]} and total number of layer is {no_of_layers}, which will freeze all layers\")\n layerf = no_of_layers-1\n print(f\"Changing freezer to {layerf}\")\n if layerf > 0:\n for child in model.children():\n for c in child.children():\n cntr+=1 \n if cntr < layerf:\n freezelayer=freezelayer+1\n for param in c.parameters():\n param.requires_grad = False\n print(f\"Number of layers freezed is {freezelayer}\")\n # if layerf == -1:\n # #print(f\"Number of layers freezed is {cntr}\")\n # for child in model.children():\n # for c in child.children():\n # for param in c.parameters():\n # param.requires_grad = True\n # print(f\"Unfreezing all layers\")\n for inputs, labels in training_loader:\n #print(len(inputs))\n #print(len(labels))\n #print(labels)\n inputs = inputs.to(device)\n labels = labels.to(device)\n batch = batch + len(inputs)\n #bs, ncrops, c, h, w = inputs.size()\n #outputs = model(input.view(-1, c, h, w))\n #outputs = model(inputs)\n \n optimizer.zero_grad()\n if modeltype == 'inception':\n outputs = model.forward(inputs)[0]\n else:\n outputs = model.forward(inputs)\n #optimizer.zero_grad()\n loss = criterion(outputs, labels)\n loss.backward()\n \n optimizer.step()\n if sch_lr != 'zero' :\n scheduler.step()\n #outputs = torch.exp(outputs)\n _, preds = torch.max(outputs, 1)\n #preds = preds + 1\n #print(preds)\n #print( labels.data)\n #print('-------------------')\n running_loss += loss.item()\n #print(running_loss,loss.item())\n running_corrects += torch.sum(preds == labels.data)\n #print(f\"Epoch {e} has accuracy of \")\n #print(torch.sum(preds == labels.data),len(inputs),int(torch.sum(preds == labels.data))/len(inputs))\n \n else:\n valbatch = 0\n with torch.no_grad():\n for val_inputs, val_labels in validation_loader:\n val_inputs = val_inputs.to(device)\n val_labels = val_labels.to(device)\n valbatch = valbatch + len(val_inputs)\n if modeltype == 'inception':\n val_outputs = model(val_inputs)[0]\n else:\n val_outputs = model(val_inputs) \n val_loss = criterion(val_outputs, val_labels)\n #val_outputs = torch.exp(val_outputs)\n _, val_preds = torch.max(val_outputs, 1)\n #val_preds = val_preds + 1\n val_running_loss += val_loss.item()\n #print(val_loss.item(),val_running_loss)\n val_running_corrects += torch.sum(val_preds == val_labels.data)\n #kappa_running +=(val_preds == val_labels.data)\n kappa_score+= quadratic_kappa(val_preds, val_labels.data)\n #print(epoch_loss) \n #print(running_corrects.float())\n #print('-----------')\n epoch_loss = running_loss/len(training_loader)\n epoch_acc = running_corrects.float()/ batch\n running_loss_history.append(epoch_loss)\n running_corrects_history.append(epoch_acc)\n val_epoch_loss = val_running_loss/len(validation_loader)\n val_kappa_score = kappa_score/len(validation_loader)\n val_epoch_acc = val_running_corrects.float()/ valbatch\n val_running_loss_history.append(val_epoch_loss)\n val_running_corrects_history.append(val_epoch_acc)\n print('epoch :', (e+1))\n print('training loss: {:.4f}, acc {:.4f} '.format(epoch_loss, epoch_acc.item()))\n print('validation loss: {:.4f}, validation acc {:.4f} '.format(val_epoch_loss, val_epoch_acc.item()))\n print('Kappa Score: {:.4f} '.format(val_kappa_score))\n print('*'*100)\n modelpth = modeltype + '.pth'\n torch.save(model, modelpth)\n return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, test_image_name,transform,image_size,modeltype='other'):\n test_image = Image.open(test_image_name).convert('RGB')\n test_image_tensor = transform(test_image)\n if torch.cuda.is_available():\n test_image_tensor = test_image_tensor.view(1, 3, image_size, image_size).cuda()\n else:\n test_image_tensor = test_image_tensor.view(1, 3, image_size, image_size)\n with torch.no_grad():\n model.eval()\n if modeltype == 'inception':\n out = model(test_image_tensor)[0]\n else:\n out = model(test_image_tensor)\n ps = torch.exp(out)\n return test_image_name,ps","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal = pd.DataFrame(columns=['id_code', 'diagnosis'])\nnewtestfinal.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def addtensorcols(val):\n return val['diagnosis1'] + val['diagnosis2']\n\ndef extractfilename(val):\n return os.path.split(val)[1]\n\ndef maxtensorval(val):\n #ps = torch.exp(val)\n #ps = F.softmax(val,dim=1)\n top_p, top_class = val.topk(1)\n return top_class\n\n\nnewtestfinal = pd.DataFrame(columns=['id_code', 'diagnosis1'])\nfor modelnum in [0]: \n #counter = 0\n print('---------------------------------------------------------------------------------')\n print('Defining model and creating data loaders for model number {0}'.format(modelnum))\n #freezer=[45,45,45,45,45,45,45,45,30,30,30,30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,25,25,25,0,0,0,0,0,0,0,40,40] #resnet152\n #freezer=[16,16,16,16,16,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] #densenet201\n freezer=[16,16,16,15,15,15,13,13,13,12,12,12,11,11,11,10,10,10,9,9,9,8,8,8,7,7,7,6,6,6,5,5,5,4,4,4,3,3,3,2,2,2,1,1,1,0,0,0]\n #freezer=[0,6,-1,0,8]\n #freezer=[16,16,16,10,10,10,0,0]\n model,criterion,optimizer,modelname,training_loader, validation_loader,image_size = definemodel(modelnum,pretrained=False,freezelonlylastlayer = 'no',\n lr=1.0)\n #test_transforms = transforms.Compose([transforms.Resize(256),\n # transforms.CenterCrop(image_size),\n # #transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n # transforms.ToTensor(),\n # transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n # ])\n test_transforms = transforms.Compose([transforms.Resize(256),\n transforms.RandomHorizontalFlip(),\n transforms.RandomRotation(10),\n transforms.CenterCrop(image_size),\n transforms.ToTensor(),\n transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n ])\n print(\"Training of model {0} started\".format(modelname))\n model.to(device)\n #optimizer = torch.optim.SGD(model.parameters(), lr=1.0, momentum=0.9, nesterov=True)\n model = modeltrainv2(model,criterion,optimizer,training_loader,validation_loader,epochs = len(freezer),\n modeltype=modelname,freezer=freezer,stepsize=4,factor=2,end_lr=0.001,\n sch_lr = 'cyclic') \n basedir = '/kaggle/images/test'\n newtest = pd.DataFrame(columns=['id_code', 'diagnosis2'])\n #destinationfolder = '../images'\n print(\"Prediction of model {0} started\".format(modelname))\n for i,row in test.iterrows():\n pathfile = basedir + '/'+row['id_code']+'.png'\n test_image_name,imagetype = predict(model, pathfile,test_transforms,image_size,modeltype=modelname)\n newtest.loc[i] = [test_image_name,imagetype]\n print(\"Prediction for model {0} completed\".format(modelname))\n newtest['id_code'] = newtest['id_code'].apply(extractfilename)\n del model,criterion,optimizer\n torch.cuda.empty_cache()\n if newtestfinal.shape[0]>0:\n newtestfinal = pd.merge(newtestfinal, newtest, on='id_code')\n newtestfinal['diagnosis1'] = newtestfinal.apply(addtensorcols, axis=1)\n newtestfinal = newtestfinal[['id_code','diagnosis1']]\n else:\n newtestfinal = newtest.copy()\n newtestfinal.columns=['id_code', 'diagnosis1']\n print(\"Training of model {0} completed\".format(modelname))\n\nnewtestfinal['diagnosis'] = newtestfinal['diagnosis1'].apply(maxtensorval)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ct = 0\n#for child in model.children():\n#ct += 1\n#if ct < 7:\n# for param in child.parameters():\n# param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model,criterion,optimizer,modelname,training_loader, validation_loader,image_size = definemodel(4,pretrained=False,freezelonlylastlayer = 'no',\n# lr=0.0001)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#layers=[]\n#lsch = list(model.children()[:-2])\n#for child in model.children():\n# for c in child.children():\n# for c1 in c.children():\n #layers.append(child)\n# for param in c.parameters():\n# print(param)\n# print('......')\n# layers.append(c)\n# layers.append(c1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#len(layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#layers[-2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for i,row in test.iterrows():\n# pathfile = basedir + '/'+row['id_code']+'.png'\n# test_image_name,imagetype = predict(model, pathfile,test_transforms,image_size,modeltype=modelname)\n# newtest.loc[i] = [test_image_name,imagetype]\n#print(\"Prediction for model {0} completed\".format(modelname))\n#newtest['id_code'] = newtest['id_code'].apply(extractfilename)\n#del model,criterion,optimizer\n#torch.cuda.empty_cache()\n#if newtestfinal.shape[0]>0:\n# newtestfinal = pd.merge(newtestfinal, newtest, on='id_code')\n# newtestfinal['diagnosis1'] = newtestfinal.apply(addtensorcols, axis=1)\n# newtestfinal = newtestfinal[['id_code','diagnosis1']]\n#else:\n# newtestfinal = newtest.copy()\n# newtestfinal.columns=['id_code', 'diagnosis1']\n#print(\"Training of model {0} completed\".format(modelname))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#newtestfinal['diagnosis'] = newtestfinal['diagnosis1'].apply(maxtensorval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = torch.tensor([4, 2, 4, 0, 2, 2, 1, 3, 1, 3, 3, 3, 4, 4, 3, 2, 0, 2, 2, 2, 4, 3, 1, 2,\n 0, 2, 1, 0, 3, 1, 3, 2, 3, 2, 1, 3, 3, 1, 4, 4, 3, 0, 2, 2, 3, 0, 1, 2,\n 1, 3, 2, 1, 0, 4, 4, 3, 4, 2, 0, 2, 0, 1, 2, 2])\nb = torch.tensor([4, 2, 0, 4, 2, 2, 1, 3, 1, 3, 3, 3, 4, 0, 3, 2, 0, 3, 2, 2, 4, 3, 1, 2,\n 0, 2, 1, 0, 3, 1, 3, 2, 3, 2, 1, 3, 3, 1, 4, 4, 3, 0, 2, 2, 3, 0, 1, 2,\n 1, 3, 2, 1, 4, 4, 4, 3, 4, 2, 0, 2, 4, 1, 2, 2])\nc = int(torch.sum(a == b))/len(b)\nc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#newtestfinal['category'] = newtestfinal['category1'].apply(maxtensorval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal['diagnosis'] = newtestfinal['diagnosis'].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal = newtestfinal[['id_code','diagnosis']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for i, x in newtestfinal.iterrows():\n# print(x['category1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal['id_code_new']=newtestfinal['id_code'].apply(lambda x:x.split('.')[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal = newtestfinal[['id_code_new','diagnosis']]\nnewtestfinal.columns = ['id_code','diagnosis']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newtestfinal.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass ImageDataset(Dataset):\n\n def __init__(self, df,\n transforms=None,\n labels_=False):\n self.labels = None\n self.transforms = None\n self.df = df\n self.imagename = np.asarray(self.df.iloc[:, 0])\n self.data_len = len(self.df.index)\n if transforms is not None:\n self.transforms = transforms\n\n def __getitem__(self, index):\n basedir = '../test/dummy/'\n image_name = basedir + self.imagename[index]\n #id_ = self.ids[index]\n img_ = Image.open(image_name).convert('RGB')\n if self.transforms is not None:\n img_ = self.transforms(img_)[:3,:,:]\n label = 0\n return (img_,image_name)\n\n def __len__(self):\n return self.data_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport base64\n\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):\n csv = df.to_csv(index=False)\n b64 = base64.b64encode(csv.encode())\n payload = b64.decode()\n html = '{title}'\n html = html.format(payload=payload,title=title,filename=filename)\n return HTML(html)\n\ncreate_download_link(newtestfinal)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}