{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom IPython import display\nfrom IPython.display import Image\nplt.rcParams.update({'figure.figsize': [8,10]})\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-02T00:31:23.798802Z","iopub.execute_input":"2022-07-02T00:31:23.799727Z","iopub.status.idle":"2022-07-02T00:31:23.812716Z","shell.execute_reply.started":"2022-07-02T00:31:23.799638Z","shell.execute_reply":"2022-07-02T00:31:23.811965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 100種高粱品種識別","metadata":{}},{"cell_type":"code","source":"import time\nimport glob\nimport os\nimport math\nimport cv2\nimport builtins\nimport copy\nos.environ['TRIDENT_BACKEND'] = 'pytorch'\nos.environ['TRIDENT_HOME'] = './trident'\n\n!pip uninstall tridentx -y\n!pip install ../input/trident/tridentx-0.7.5-py3-none-any.whl --upgrade","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:23.818457Z","iopub.execute_input":"2022-07-02T00:31:23.819046Z","iopub.status.idle":"2022-07-02T00:31:36.086236Z","shell.execute_reply.started":"2022-07-02T00:31:23.819008Z","shell.execute_reply":"2022-07-02T00:31:36.085059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import trident as T\nfrom trident import *\nfrom trident.models import efficientnet\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:36.089244Z","iopub.execute_input":"2022-07-02T00:31:36.089566Z","iopub.status.idle":"2022-07-02T00:31:39.400464Z","shell.execute_reply.started":"2022-07-02T00:31:36.089526Z","shell.execute_reply":"2022-07-02T00:31:39.399400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf=pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv').dropna()\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:39.403445Z","iopub.execute_input":"2022-07-02T00:31:39.403933Z","iopub.status.idle":"2022-07-02T00:31:39.449662Z","shell.execute_reply.started":"2022-07-02T00:31:39.403891Z","shell.execute_reply":"2022-07-02T00:31:39.448963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classnames=df.cultivar.unique().tolist()\nclassnames=list(sorted(classnames))\nprint(classnames)","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:39.450944Z","iopub.execute_input":"2022-07-02T00:31:39.451384Z","iopub.status.idle":"2022-07-02T00:31:39.459313Z","shell.execute_reply.started":"2022-07-02T00:31:39.451347Z","shell.execute_reply":"2022-07-02T00:31:39.458418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nall_images=glob.glob('../input/sorghum-id-fgvc-9/train_images/*.*g')\nprint(len(all_images))\n\nimages=[]\nlabels=[]\n\nfor index, row in df.iterrows():\n    impath='../input/sorghum-id-fgvc-9/train_images/'+row['image']\n    if impath in all_images:\n        images.append(impath)\n        labels.append(classnames.index(row['cultivar']))\n        \nprint(len(images))\nprint(len(labels))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:39.460512Z","iopub.execute_input":"2022-07-02T00:31:39.461334Z","iopub.status.idle":"2022-07-02T00:31:47.514419Z","shell.execute_reply.started":"2022-07-02T00:31:39.461295Z","shell.execute_reply":"2022-07-02T00:31:47.513621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\ncl=CLAHE()\ndef multi_scale_colors(img,spec=None):\n    #print(img.shape)\n    \n    img=cl(img)\n    img1=cv2.resize(img.copy(),(240,240))\n    \n \n    idx=random.choice(list(range(4)))\n    #print('idx:',idx,'{0}:{1},{2}:{3}'.format((idx//2)*512,(idx//2+1)*512,(idx%2)*512,(idx%2+1)*512))\n    crop_image=img.copy()[(idx//2)*512:(idx//2+1)*512,(idx%2)*512:(idx%2+1)*512,:]\n    #print(crop_image.shape)\n    img2=cv2.resize(crop_image,(240,240), interpolation=cv2.INTER_AREA)\n    \n    idxes=random.choices(list(range(16)),k=4)\n    #print('idxes:',idxes)\n    img3=cv2.resize(img.copy()[(idxes[0]//4)*256:(idxes[0]//4+1)*256,(idxes[0]%4)*256:(idxes[0]%4+1)*256,:],(240,240), interpolation=cv2.INTER_AREA)\n    img4=cv2.resize(img.copy()[(idxes[1]//4)*256:(idxes[1]//4+1)*256,(idxes[1]%4)*256:(idxes[1]%4+1)*256,:],(240,240), interpolation=cv2.INTER_AREA)\n    \n    img5=cv2.cvtColor(cv2.resize(img.copy()[(idxes[2]//4)*256:(idxes[2]//4+1)*256,(idxes[2]%4)*256:(idxes[2]%4+1)*256,:],(240,240), interpolation=cv2.INTER_AREA),cv2.COLOR_RGB2HSV)\n    img9=cv2.cvtColor(cv2.resize(img.copy()[(idxes[3]//4)*256:(idxes[3]//4+1)*256,(idxes[3]%4)*256:(idxes[3]%4+1)*256,:],(240,240), interpolation=cv2.INTER_AREA),cv2.COLOR_BGR2YCR_CB)\n    \n    idxes=random.choices(list(range(64)),k=3)\n    img6=cv2.resize(img.copy()[(idxes[0]//8)*128:(idxes[0]//8+1)*128,(idxes[0]%8)*128:(idxes[0]%8+1)*128,:],(240,240), interpolation=cv2.INTER_AREA)\n    img7=cv2.resize(img.copy()[(idxes[1]//8)*128:(idxes[1]//8+1)*128,(idxes[1]%8)*128:(idxes[1]%8+1)*128,:],(240,240), interpolation=cv2.INTER_AREA)\n    idx=random.choice(list(range(64)))\n    img8=cv2.cvtColor(cv2.resize(img.copy()[(idxes[2]//8)*128:(idxes[2]//8+1)*128,(idxes[2]%8)*128:(idxes[2]%8+1)*128,:],(240,240), interpolation=cv2.INTER_AREA),cv2.COLOR_RGB2HSV)\n    image_lists=[img1,img2,img3,img4,img5,img6,img7,img8,img9]\n    random.shuffle(image_lists)\n    new_img=np.concatenate([np.concatenate(image_lists[0:3],axis=1),np.concatenate(image_lists[3:6],axis=1),np.concatenate(image_lists[6:9],axis=1)],axis=0)\n    \n    return new_img\n\n\n\n    \ndisplay.display(array2image(multi_scale_colors(cl(image2array('../input/sorghum-id-fgvc-9/train_images/2017-06-01__10-26-27-479.png').astype(np.uint8)))))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:47.515731Z","iopub.execute_input":"2022-07-02T00:31:47.516331Z","iopub.status.idle":"2022-07-02T00:31:48.067636Z","shell.execute_reply.started":"2022-07-02T00:31:47.516288Z","shell.execute_reply":"2022-07-02T00:31:48.066132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds1=ImageDataset(images,object_type=ObjectType.rgb,symbol='images')\nds2=LabelDataset(labels,object_type=ObjectType.classification_label,symbol='target')\n#ds3=ImageDataset(images,object_type=ObjectType.rgb,symbol='images2')\n\nds2.binding_class_names(class_names=classnames)\nprint(ds2.class_names)\ndata_provider=DataProvider(traindata=Iterator(data=ds1,label=ds2,batch_size=4))\n#data_provider=DataProvider(traindata=Iterator(data=ds1,label=ZipDataset(ds2,ds3),batch_size=4))\ndata_provider.paired_transform_funcs=[\n    RandomTransform(rotation_range=10, zoom_range=(0.9,1.2), shift_range=0.05, shear_range=0.1, random_flip=0.2,keep_prob=0.3,border_mode='zero'), \n    ]\n\n#    RandomBlur(ksize_range=(3,11),keep_prob=0.7),\ndata_provider.image_transform_funcs = [\n    RandomAdjustGamma(gamma_range=(0.6,1.1)),\n    RandomAdjustSaturation(value_range=(0.8, 1.6)),\n    RandomAdjustContrast(value_range=(0.8, 1.4)),\n    multi_scale_colors,\n    AutoLevel(),\n    SaltPepperNoise(prob=0.002),  # 椒鹽噪音\n    Normalize(127.5, 127.5)]","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:48.068815Z","iopub.execute_input":"2022-07-02T00:31:48.069133Z","iopub.status.idle":"2022-07-02T00:31:49.823828Z","shell.execute_reply.started":"2022-07-02T00:31:48.069084Z","shell.execute_reply":"2022-07-02T00:31:49.822907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def space_to_depth(x:np.ndarray, block_size=3):\n    sq_size=block_size*block_size\n    \n    if len(x.shape)==4 and  x.shape[1]==3:\n        new_tensors=[]\n        for i in range(x.shape[0]):\n            new_tensors.append(space_to_depth(x[i]))\n        new_tensors=stack(new_tensors,axis=0)\n            \n    elif len(x.shape)==3:  \n        new_tensors=[]\n        if len(x.shape)==3 and x.shape[0]>x.shape[-1]:\n            x=x.transpose([2,0,1])\n        for i in range(block_size*block_size):\n            new_tensors.append(x[:,(i//block_size)*240:(i//block_size+1)*240,(i%block_size)*240:(i%block_size+1)*240])\n            \n        new_tensors=stack(new_tensors,axis=0)\n    return new_tensors\n        \n\n\narr=multi_scale_colors(cl(image2array('../input/sorghum-id-fgvc-9/train_images/2017-06-01__10-26-27-479.png').astype(np.uint8)))\nprint(arr.shape)\narr=space_to_depth(to_tensor(image_backend_adaption(arr)), block_size=3)\nprint(arr.shape)\n# arr1=space_to_depth(to_tensor(image_backend_adaption(arr)), block_size=3)\n# print(arr0.shape)\ndisplay.display(array2image(to_numpy(arr)[0].transpose([1,2,0]).astype(np.uint8)))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:49.825022Z","iopub.execute_input":"2022-07-02T00:31:49.825284Z","iopub.status.idle":"2022-07-02T00:31:49.973800Z","shell.execute_reply.started":"2022-07-02T00:31:49.825250Z","shell.execute_reply":"2022-07-02T00:31:49.973067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_provider.preview_images()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:49.975144Z","iopub.execute_input":"2022-07-02T00:31:49.975790Z","iopub.status.idle":"2022-07-02T00:31:52.132051Z","shell.execute_reply.started":"2022-07-02T00:31:49.975749Z","shell.execute_reply":"2022-07-02T00:31:52.130941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#_images,_labels=data_provider.next()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.135221Z","iopub.execute_input":"2022-07-02T00:31:52.135457Z","iopub.status.idle":"2022-07-02T00:31:52.139099Z","shell.execute_reply.started":"2022-07-02T00:31:52.135427Z","shell.execute_reply":"2022-07-02T00:31:52.138399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport gc\n# torch.cuda.synchronize()\n# torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.140415Z","iopub.execute_input":"2022-07-02T00:31:52.141419Z","iopub.status.idle":"2022-07-02T00:31:52.329114Z","shell.execute_reply.started":"2022-07-02T00:31:52.141356Z","shell.execute_reply":"2022-07-02T00:31:52.328370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TanhExp(Layer):\n\n    def __init__(self,keep_output=False, name=None):\n        super(TanhExp, self).__init__(keep_output=keep_output,name=name)\n        self._built = True\n\n    def forward(self, x, **kwargs):\n\n        return clip(x*torch.tanh(torch.exp(x)),-2,2)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.333595Z","iopub.execute_input":"2022-07-02T00:31:52.334420Z","iopub.status.idle":"2022-07-02T00:31:52.341864Z","shell.execute_reply.started":"2022-07-02T00:31:52.334384Z","shell.execute_reply":"2022-07-02T00:31:52.341049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n    \nclass SorghumNetV1(Layer):\n    def __init__(self, n_classes=100):\n        super(SorghumNetV1, self).__init__()\n        \n        _effb1=efficientnet.EfficientNetB1(pretrained=True,input_shape=(3,240,240),include_top=False)\n        _effb1.model.top_conv.trainable=True\n        _effb1.model.block7b.trainable=True\n        _effb1.model.block7b.dropout_rate=0.2\n        _effb1.model.top_conv.activation=TanhExp()\n\n        self.n_classes=n_classes\n        self.backbone =_effb1.model\n        self.agg=Sequential(\n            Reshape((9,1280, 8, 8)),\n            Aggregation(mode='mean',axis=1,keepdims=False),\n            SeparableConv2d_Block((3,3),depth_multiplier=1,strides=1,auto_pad=True,use_bias=False,activation=TanhExp(),normalization='bn'),\n            GlobalAvgPool2d(),\n        )\n\n        self.decoder=Dense(n_classes,activation=SoftMax())\n     \n    def forward(self, x):\n        new_x=space_to_depth(x, block_size=3)\n        B,N,C,H,W=new_x.shape\n        new_x=new_x.reshape((B*N,C,H,W))\n    \n        return self.decoder(self.agg(self.backbone(new_x)))\n    \n    \n    \nclass SorghumNetV2(Layer):\n    def __init__(self, n_classes=100):\n        super(SorghumNetV2, self).__init__()\n        \n        _effb1=efficientnet.EfficientNetB1(pretrained=True,input_shape=(3,240,240),include_top=False)\n        _effb1.model.trainable=True\n        _effb1.model.block7b.dropout_rate=0.2\n        _effb1.model.top_conv.activation=TanhExp()\n\n        self.n_classes=n_classes\n        self.backbone =_effb1.model\n        self.agg=Sequential(\n            Reshape((9,1280, 8, 8)),\n            Aggregation(mode='max',axis=1,keepdims=False),\n            SeparableConv2d_Block((3,3),depth_multiplier=1,strides=1,auto_pad=True,use_bias=False,activation=TanhExp(),normalization='bn'),\n            ShortCut(\n                Identity(),\n                Sequential(\n                GlobalAvgPool2d(),\n                Reshape((1280,1,1)),\n                Conv2d((1,1),num_filters=100,use_bias=False,activation=TanhExp()),\n                Conv2d((1,1),num_filters=1280,use_bias=False,activation=Sigmoid())\n                ),mode='dot'\n            ),\n            GlobalAvgPool2d(),\n        )\n        \n    \n     \n        self.decoder=Dense(n_classes,weight_norm='l2')\n    \n    \n    def forward(self, x):\n        new_x=space_to_depth(x, block_size=3)\n        B,N,C,H,W=new_x.shape\n        new_x=new_x.reshape((B*N,C,H,W))\n    \n      \n        return self.decoder(self.agg(self.backbone(new_x)))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.345086Z","iopub.execute_input":"2022-07-02T00:31:52.345292Z","iopub.status.idle":"2022-07-02T00:31:52.364215Z","shell.execute_reply.started":"2022-07-02T00:31:52.345267Z","shell.execute_reply":"2022-07-02T00:31:52.363351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sorghumnet_v1=Model(input_shape=(3,720,720),output=SorghumNetV2(100))\n# #sorghumnet_v1.load_model('../input/sorghum-100-identification/Models/sorghumnet_v1_b1.pth')\n# #sorghumnet_v1.load_model('./Models/sorghumnet_v1_b1.pth')\n# sorghumnet_v1.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.366658Z","iopub.execute_input":"2022-07-02T00:31:52.368215Z","iopub.status.idle":"2022-07-02T00:31:52.379141Z","shell.execute_reply.started":"2022-07-02T00:31:52.368176Z","shell.execute_reply":"2022-07-02T00:31:52.378369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"自蒸餾，顧名思義自己蒸餾自己，嚴格說起來是指，老師正是幾個批次前、並切換成為評估模式的自己，由於在評估模式中，像是dropout, add_noise等操作將會失效，因此數據會呈現較為乾淨的效果，因此雖然是用相同權重。光是關閉droupout，就讓老師模型相對於同樣權重還原回來的學生模型的效度略勝一籌。也因此我們需要把老師模型權重凍結、設定為評估模式。","metadata":{}},{"cell_type":"markdown","source":"<img src='https://docs.google.com/uc?export=download&id=1hfxk-uZ_m9e5yueAFX9y81gjEa8y2IqG'/>","metadata":{}},{"cell_type":"code","source":"sorghumnet_v2_teacher=Model(input_shape=(3,720,720),output=SorghumNetV2(100))\n\n#\n\nif os.path.exists('./Models/sorghumnet_v2_b1.pth'):\n    sorghumnet_v2_teacher.load_model('./Models/sorghumnet_v2_b1.pth')\nelif os.path.exists('../input/sorghum-100-identification-revised2/Models/sorghumnet_v2_b1.pth.tar'):\n    sorghumnet_v2_teacher.load_model('../input/sorghum-100-identification-revised2/Models/sorghumnet_v2_b1.pth.tar')\n\n\nsorghumnet_v2_teacher.model.eval()\nsorghumnet_v2_teacher.trainable=False\n\nsorghumnet_v2_teacher.model.agg[0].keep_output=True\nsorghumnet_v2_teacher.summary()\nsorghumnet_v2_teacher.training_context['stop_updates']=10000\nsorghumnet_v2_teacher.training_context['training_name']='teacher'\n#sorghumnet_v2.output_fn =prepare_output","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:52.380194Z","iopub.execute_input":"2022-07-02T00:31:52.382140Z","iopub.status.idle":"2022-07-02T00:31:55.488109Z","shell.execute_reply.started":"2022-07-02T00:31:52.382100Z","shell.execute_reply":"2022-07-02T00:31:55.487294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"學生模型則是除了從標籤中學習正確答案之外，","metadata":{}},{"cell_type":"code","source":"sorghumnet_v2=Model(input_shape=(3,720,720),output=SorghumNetV2(100))\nif os.path.exists('./Models/sorghumnet_v2_b1.pth'):\n    sorghumnet_v2.load_model('./Models/sorghumnet_v2_b1.pth')\nelif os.path.exists('../input/sorghum-100-identification-revised2/Models/sorghumnet_v2_b1.pth.tar'):\n    sorghumnet_v2.load_model('../input/sorghum-100-identification-revised2/Models/sorghumnet_v2_b1.pth.tar')\n\n\n#sorghumnet_v2.load_model('./Models/sorghumnet_v2_b1.pth')\nsorghumnet_v2.trainable=True\nsorghumnet_v2.training_context['training_name']='student'\nsorghumnet_v2.model.agg[0].keep_output=True\nsorghumnet_v2.summary()\n#sorghumnet_v2.output_fn =prepare_output","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:55.489601Z","iopub.execute_input":"2022-07-02T00:31:55.489917Z","iopub.status.idle":"2022-07-02T00:31:57.524043Z","shell.execute_reply.started":"2022-07-02T00:31:55.489877Z","shell.execute_reply":"2022-07-02T00:31:57.523206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProductLoss(Layer):\n    def __init__(self, scale=32.0, margin=0.80, easy_margin=False, num_filters= 100,name='ArcMarginProductLoss'):\n        super(ArcMarginProductLoss, self).__init__()\n        self._name=name\n        self.num_filters=num_filters\n        self.scale = scale\n        self.m = margin\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(margin)\n        self.sin_m = math.sin(margin)\n\n        # make the function cos(theta+m) monotonic decreasing while theta in [0°,180°]\n        self.th = math.cos(math.pi - margin)\n        self.mm = math.sin(math.pi - margin) * margin\n        self.base_loss=CrossEntropyLoss(reduction='mean')\n\n\n    def forward(self,output, target,**kwargs):\n        # cos(theta)\n        try:\n            cosine=l2_normalize(output)\n            \n            # cos(theta + m)\n            sine = sqrt(1.0 - pow(cosine, 2))\n            phi = cosine * self.cos_m - sine * self.sin_m\n\n            if self.easy_margin:\n                phi = where(cosine > 0, phi, cosine)\n            else:\n                phi = where((cosine - self.th) > 0, phi, cosine - self.mm)\n\n            one_hot = zeros_like(cosine,requires_grad=True)\n            one_hot.scatter(1, target.view(-1, 1), 1)\n\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n            output = output * self.scale\n        except Exception as e:\n            print(e)\n            PrintException()\n\n        loss = self.base_loss(output, target)\n        return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.525591Z","iopub.execute_input":"2022-07-02T00:31:57.526044Z","iopub.status.idle":"2022-07-02T00:31:57.547372Z","shell.execute_reply.started":"2022-07-02T00:31:57.526006Z","shell.execute_reply":"2022-07-02T00:31:57.546507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import manifold\nfrom tqdm import  tqdm\n\n \n#將arcface視覺化\ndef visualize(training_context):\n    features=[]\n    labels=[]\n    preds=[]\n    model=training_context['current_model']\n    epoch=training_context['current_epoch']\n    model.eval()\n    print('switch to evaluation.')\n#         if epoch==1:\n  \n#             model.block6e.trainable=True\n#             model.block6d.trainable=True\n#             model.block6c.trainable=True\n#             model.block6b.trainable=True\n#         elif epoch==11:\n#             model.block6a.trainable=True\n#         elif epoch==13:\n#             model.block5d.trainable=True\n#         elif epoch==15:\n#             model.block5c.trainable=True\n\n\n    NUM_COLORS = 100\n    cm = plt.get_cmap('gist_rainbow')\n    for i in tqdm(range(50)):\n        _images,_labels=data_provider2.next()\n        _result=to_numpy(argmax(model(to_tensor(_images)),axis=1))\n\n        _features=model.agg[3].branch2[0].output\n\n        for k in range(len(_images)):\n            features.append(to_numpy(l2_normalize(_features[k])))\n            labels.append(_labels[k])\n            preds.append(_result[k])\n    print('features',len(features),'labels',len(labels))\n    labels=np.array(labels)\n    features=np.array(features)\n    preds=np.array(preds)\n\n    print('accuracy:{0:.3%}'.format(np.equal(preds,labels).astype(np.float32).mean()))\n\n\n    #利用TSNE降維成2維後，繪製成散布圖\n\n    fig = plt.figure(figsize=(12,12))\n    ax1= fig.add_subplot(1, 1, 1)\n    tsne2 = manifold.TSNE(n_components=2, init='pca', random_state=0)  # 利用t-sne將512特徵向量降維至2\n    print('tsne 訓練開始')\n    features_tsne2 = tsne2.fit_transform(features) \n    #features_tsne2=l2_normalize(features_tsne2)\n    print('tsne 訓練結束')\n    for i in range(100):\n        x_i = features_tsne2[:,0][labels==i]\n        y_i = features_tsne2[:,1][labels==i]\n        ax1.scatter(x_i,y_i,s=20,marker='o',c=cm(i//3*3.0/NUM_COLORS))\n\n    model.train()\n    plt.legend(classnames, loc = 'upper right')\n    plt.title('epoch {0}'.format(epoch))\n    make_dir_if_need('results')\n    plt.savefig('results/epoch{0}.jpg'.format(epoch), bbox_inches='tight')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.548861Z","iopub.execute_input":"2022-07-02T00:31:57.549434Z","iopub.status.idle":"2022-07-02T00:31:57.624900Z","shell.execute_reply.started":"2022-07-02T00:31:57.549395Z","shell.execute_reply":"2022-07-02T00:31:57.624181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#visualize(sorghumnet_v2.training_context)","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.626545Z","iopub.execute_input":"2022-07-02T00:31:57.626802Z","iopub.status.idle":"2022-07-02T00:31:57.637420Z","shell.execute_reply.started":"2022-07-02T00:31:57.626768Z","shell.execute_reply":"2022-07-02T00:31:57.633764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sorghumnet_v2.load_model('./Models/sorghumnet_v2_b1.pth')\n#sorghumnet_v2.load_model('../input/sorghum-100-identification/Models/sorghumnet_v2_b1.pth.tar')","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.640366Z","iopub.execute_input":"2022-07-02T00:31:57.640751Z","iopub.status.idle":"2022-07-02T00:31:57.645150Z","shell.execute_reply.started":"2022-07-02T00:31:57.640706Z","shell.execute_reply":"2022-07-02T00:31:57.644325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"對比學習的重要精神: \n1. 假設最極端的狀況，每一張圖片都是獨一無二，每個圖片自己是自己的正樣本，全世界的其他圖片都是它的負樣本。\n2. 如何解決單一圖片只有一個正樣本的問題。\\=\\>數據增強 (尤其是空間以及色彩變化)\n3. 但是這樣會造成必須讓一張圖片透過數據增強產生多個版本圖片後，再送入模型以產生對應的特徵圖，這樣會耗費數倍的算力。\n4. 別忘了我們一開始設計的九宮格，正是透過不同的尺度，不同的色彩座標來產生的，它正是內部的數據增強，所以直接取得內部的特徵圖即可不須額外新樣本送入模型。\n","metadata":{}},{"cell_type":"markdown","source":"<img src='https://docs.google.com/uc?export=download&id=1H9EYFLYu365J58QRkfSNWNGusgAUPuw2'/>","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\n#batch=8  8*9*256*256\n#對比損失\n#在這個模型的設計等於每個案例有9種視角(多尺度、不同色彩系統)，這跟對比學習的數據增強是一樣概念\n#所以同一個案例不同視角的特徵應該一致\n#不同案例特徵應該要有差異\ndef contractive_loss(features):\n    temperature=0.5\n    #把批次數*9作為基礎產生標籤   0,1,2,3...0,1,2,3....  (重複9次)\n    labels = torch.cat([torch.arange(features.shape[0]) for i in range(9)], dim=0)\n    #如果是同個案例者為1其餘為零\n    labels = (labels.unsqueeze(0) == labels.unsqueeze(1)).float()\n    #把(B,9,128,8,8)變成(B*9,128,64)然後透過平均變成(B*9,128)\n    features=features.reshape((features.size(0)*features.size(1),features.size(3),-1)).mean(-1)\n    #標準化\n    features = F.normalize(features, dim=1)\n    #計算相似性矩陣\n    similarity_matrix = torch.matmul(features, features.T)\n\n    \n    #產生對角線遮罩(自己對自己必定為1，需要排除)\n    mask = torch.eye(labels.shape[0], dtype=torch.bool)\n    #排除對角線\n    labels = labels[~mask].view(labels.shape[0], -1)\n    similarity_matrix = similarity_matrix[~mask].view(similarity_matrix.shape[0], -1)\n    \n    # 選取正樣本\n    positives = similarity_matrix[labels.bool()].view(labels.shape[0], -1)\n\n    # 選取負樣本\n    negatives = similarity_matrix[~labels.bool()].view(similarity_matrix.shape[0], -1)\n    #print(positives.shape,negatives.shape)\n    #疊合\n    logits = torch.cat([positives, negatives], dim=1).reshape(-1)\n    labels = torch.cat([ones_like(positives), zeros_like(negatives)], dim=1).long().reshape(-1)\n    #labels = torch.zeros(logits.shape[0], dtype=torch.long).to(get_device())\n\n    logits = logits /temperature\n    return binary_cross_entropy(logits,labels).mean()\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.646937Z","iopub.execute_input":"2022-07-02T00:31:57.647523Z","iopub.status.idle":"2022-07-02T00:31:57.662600Z","shell.execute_reply.started":"2022-07-02T00:31:57.647482Z","shell.execute_reply":"2022-07-02T00:31:57.661650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"_OHEM (online hard example mining，線上困難樣本挖掘)_ 是在建模過程中收集預測錯誤之樣本，並進行重放，讓原本較為稀有的困難樣本得以較高的頻率出現，也增進了模型處理這類困難樣本的能力。","metadata":{}},{"cell_type":"code","source":"from trident.reinforcement.utils import Rollout\ncxt=get_session()\ncxt.rollout=Rollout(only_cpu=True)\ncxt.loss_fn=CrossEntropyLoss()\n\n\n\n#每個批次把暫存的特徵圖保存至training_context['train_data']\ndef get_features(training_context):\n    cxt=get_session()\n    model=training_context['current_model']\n    data=training_context['train_data']\n    data['features']=model.agg[0].output\n    step=training_context['steps']\n    if training_context['training_name']=='teacher':\n        cxt.features=model.agg[0].output.copy()\n        if step>0 and step%100==0:\n            model.load_state_dict(load('./Models/sorghumnet_v2_b1.pth').state_dict())\n            \n        \n\n    \n\n#收集困難樣本\ndef collect_hard_examples(training_context):\n    model=training_context['current_model']\n    data=training_context['train_data']\n    images=data['images']\n    target=data['target']\n    features=data['features']\n    pred=argmax(exp(data['output']),1)\n    \n    mask=not_equal(pred,target)\n\n    steps=training_context['steps']\n    for i in range(images.size(0)):\n        if mask[i].item==True:\n            if len(cxt.rollout)<=20 or (len(cxt.rollout)>20 and steps%10!=7):\n                cxt.rollout.collect('images',images[i])\n                cxt.rollout.collect('target',target[i])\n                cxt.rollout.collect('features',features[i])\n                #print('hard samples:{0} iou:{1:.3%} landmark_rmse:{2:.3%}'.format(len(cxt.rollout),_iou.item(),_landmark_rmse.item()))\n\n                if (len(cxt.rollout)>0 and len(cxt.rollout)%20)or (len(cxt.rollout)<10)==0:\n                    print(yellow_color('hard samples:{0}'.format(len(cxt.rollout))))\n\n                if len(cxt.rollout)>300:\n                    cxt.rollout.housekeeping(num_samples=100)\n           \n                   \n            \n#將訓練樣本局部抽換成困難樣本\ndef replace_hard_examples(training_context):\n    data=training_context['train_data']\n    steps=training_context['steps']\n    with torch.no_grad():      \n        if len(cxt.rollout)>200 and steps%10==7:\n            images=data['images']\n            target=data['target']\n            features=data['features']\n            hards_samples=cxt.rollout.get_samples(int_shape(images)[0],recall_last=False)\n            data['images']=stack(hards_samples['images'],0).float().detach()\n            data['target']=stack(hards_samples['target'],0).long().detach()\n            data['features']=stack(hards_samples['features'],0).long().detach()\n            torch.cuda.synchronize()\n            torch.cuda.empty_cache()\n            gc.collect()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.665686Z","iopub.execute_input":"2022-07-02T00:31:57.665908Z","iopub.status.idle":"2022-07-02T00:31:57.763429Z","shell.execute_reply.started":"2022-07-02T00:31:57.665884Z","shell.execute_reply":"2022-07-02T00:31:57.762557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def knowledge_distillation_loss(features):\n    _dtype=features.dtype\n    cxt=get_session()\n    student_features=features.float()\n    if hasattr(cxt,'features'):\n        teacher_features=cxt.features.float()\n        loss1=abs(student_features-teacher_features.detach()).mean().to(_dtype)\n        return loss1\n    else:\n        return 0\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.764718Z","iopub.execute_input":"2022-07-02T00:31:57.765085Z","iopub.status.idle":"2022-07-02T00:31:57.773189Z","shell.execute_reply.started":"2022-07-02T00:31:57.765049Z","shell.execute_reply":"2022-07-02T00:31:57.772418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sorghumnet_v1.load_model('./Models/sorghumnet_v1_b1.pth')\n#sorghumnet_v2.load_model('./Models/sorghumnet_v2_b1.pth')\n\n#sorghumnet_v1.load_model('../input/sorghum-100-identification/Models/sorghumnet_v1_b1.pth.tar')\n#sorghumnet_v2.load_model('../input/sorghum-100-identification/Models/sorghumnet_v2_b1.pth.tar')\n#sorghumnet_v2.model.load_state_dict(sorghumnet_v1.model.state_dict(),False)\n\nsorghumnet_v2_teacher.with_optimizer(optimizer=DiffGrad,lr=1e-8,betas=(0.9, 0.999))\\\n    .with_metric(accuracy)\\\n    .with_metric(accuracy,topk=3,name='top3_accuracy',print_only=True)\\\n    .with_regularizer('l2',1e-7) \\\n    .with_model_save_path('./Models/sorghumnet_v2_teacher.pth')\\\n    .with_learning_rate_scheduler(CosineLR(min_lr=1e-5,period=1000))\\\n    .trigger_when(when='on_loss_calculation_start',frequency=1,unit='batch',action=get_features)\n\n#ArcMarginProductLoss 間距加大 縮放變小\n#加入contractive_loss對比損失\nsorghumnet_v2.with_optimizer(optimizer=DiffGrad,lr=2e-4,betas=(0.9, 0.999),gradient_centralization='all')\\\n    .with_loss(ArcMarginProductLoss(scale=16.0, margin=1.5, easy_margin=True, num_filters=100),as_metric=True)\\\n    .with_loss(contractive_loss,loss_weight=0.2,as_metric=True)\\\n    .with_loss(knowledge_distillation_loss,as_metric=True)\\\n    .with_metric(accuracy)\\\n    .with_metric(accuracy,topk=3,name='top3_accuracy',print_only=True)\\\n    .with_regularizer('l2',1e-7) \\\n    .with_model_save_path('./Models/sorghumnet_v2_b1.pth')\\\n    .with_accumulate_grads(5)\\\n    .with_learning_rate_scheduler(CosineLR(min_lr=1e-5,period=1000))\\\n    .trigger_when(when='on_loss_calculation_start',frequency=1,unit='batch',action=get_features)\\\n    .trigger_when(when='on_batch_end',frequency=1,unit='batch',action=collect_hard_examples)\\\n    .trigger_when(when='on_data_received',frequency=1,unit='batch',action=replace_hard_examples)\\\n    .with_automatic_mixed_precision_training()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.774594Z","iopub.execute_input":"2022-07-02T00:31:57.775026Z","iopub.status.idle":"2022-07-02T00:31:57.819291Z","shell.execute_reply.started":"2022-07-02T00:31:57.774977Z","shell.execute_reply":"2022-07-02T00:31:57.818435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplan=TrainingPlan()\\\n    .add_training_item(sorghumnet_v2_teacher,name='teacher')\\\n    .add_training_item(sorghumnet_v2,name='student')\\\n    .with_data_loader(data_provider)\\\n    .repeat_epochs(100)\\\n    .with_batch_size(8)\\\n    .print_progress_scheduling(20,unit='batch')\\\n    .out_sample_evaluation_scheduling(frequency=100,unit='batch')\\\n    .display_loss_metric_curve_scheduling(frequency=100,unit='batch',imshow=True)\\\n    .save_model_scheduling(20,unit='batch')\n\n\nplan.start_now()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T00:31:57.820632Z","iopub.execute_input":"2022-07-02T00:31:57.821074Z","iopub.status.idle":"2022-07-02T04:46:06.260542Z","shell.execute_reply.started":"2022-07-02T00:31:57.821037Z","shell.execute_reply":"2022-07-02T04:46:06.259807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#清除gpu快取\nimport torch\nif is_gpu_available():\n    torch.cuda.synchronize()\n    torch.cuda.empty_cache()\n    gc.collect()\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:46:06.261829Z","iopub.execute_input":"2022-07-02T04:46:06.263198Z","iopub.status.idle":"2022-07-02T04:46:06.721102Z","shell.execute_reply.started":"2022-07-02T04:46:06.263138Z","shell.execute_reply":"2022-07-02T04:46:06.720060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#讀取大會給的範例格式是否一致\nf=open('../input/sorghum-id-fgvc-9/sample_submission.csv','r',encoding='utf-8-sig')\nrows=f.readlines()\nprint(rows[:5])","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:46:06.724928Z","iopub.execute_input":"2022-07-02T04:46:06.725161Z","iopub.status.idle":"2022-07-02T04:46:06.767469Z","shell.execute_reply.started":"2022-07-02T04:46:06.725135Z","shell.execute_reply":"2022-07-02T04:46:06.766673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_imgs=glob.glob('../input/sorghum-id-fgvc-9/test/*.*g')\nprint(len(test_imgs))\ntest_imgs=list(sorted(test_imgs))\nsubmission_dict=OrderedDict()\nfor im_path in test_imgs:\n    folder,filename,ext=split_path(im_path)\n    submission_dict[filename+ext]=None\n    \n\n\n\n\nds1_test=ImageDataset(test_imgs,object_type=ObjectType.rgb,symbol='images')\nds2_test=ImageDataset(test_imgs,object_type=ObjectType.image_path,symbol='images_path')\n\n\ndata_provider_test=DataProvider(traindata=Iterator(data=ds1_test,label=ds2_test,batch_size=4))\n\n\ndata_provider_test.image_transform_funcs = [\n    multi_scale_colors,\n    AutoLevel(),\n    Normalize(127.5, 127.5)]","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:46:06.768599Z","iopub.execute_input":"2022-07-02T04:46:06.769353Z","iopub.status.idle":"2022-07-02T04:46:07.902885Z","shell.execute_reply.started":"2022-07-02T04:46:06.769303Z","shell.execute_reply":"2022-07-02T04:46:07.902011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorghumnet_v2.eval()\nn=0\nfor i,(test_images,test_images_path) in enumerate(data_provider_test):\n    result=argmax(sorghumnet_v2(to_tensor(test_images).to(get_device())),1)\n    if is_gpu_available():\n        torch.cuda.synchronize()\n        torch.cuda.empty_cache()\n        gc.collect()\n    for k in range(len(result)):\n        folder,filename,ext=split_path(test_images_path[k])\n        if filename+ext not in submission_dict:\n            print(filename+ext,' not in submission_dict!')\n        else:\n            submission_dict[filename+ext]=classnames[result[k].item()]\n        \n    n+=len(test_images)\n    if n%100==0:\n        print(n)\n    if n>=len(test_imgs) and len([k for k,v in submission_dict.items() if v is None])==0:\n        break\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:46:07.905094Z","iopub.execute_input":"2022-07-02T04:46:07.905845Z","iopub.status.idle":"2022-07-02T06:29:56.124687Z","shell.execute_reply.started":"2022-07-02T04:46:07.905798Z","shell.execute_reply":"2022-07-02T06:29:56.123911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_rows=['filename,cultivar\\n']\nfor k,v in submission_dict.items():\n    submission_rows.append('{0},{1}\\n'.format(k,v))\n#確認提交檔筆數\nprint(len(submission_rows))\n#寫入提交檔\nmake_dir_if_need('./results')\nwith open('./results/submission.csv','w',encoding='utf-8-sig') as f:\n    f.writelines(submission_rows)\n#再讀取一次確認何大會給的範例格式是否一致\nfr=open('./results/submission.csv','r',encoding='utf-8-sig')\nrows=fr.readlines()\nprint(rows[:3])","metadata":{"execution":{"iopub.status.busy":"2022-07-02T06:39:41.541787Z","iopub.execute_input":"2022-07-02T06:39:41.543090Z","iopub.status.idle":"2022-07-02T06:39:41.591963Z","shell.execute_reply.started":"2022-07-02T06:39:41.543047Z","shell.execute_reply":"2022-07-02T06:39:41.591204Z"},"trusted":true},"execution_count":null,"outputs":[]}]}