{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":17018,"databundleVersionId":799209,"sourceType":"competition"}],"dockerImageVersionId":30554,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import transformers\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport datasets\nfrom tqdm import tqdm\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader\nfrom datasets import load_dataset\nfrom torchvision.transforms.v2 import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor\nfrom torchvision.transforms.v2 import Compose, ColorJitter, ToTensor\nfrom torchvision.transforms.v2 import (\n CenterCrop,\n Compose,\n Normalize,\n RandomHorizontalFlip,\n RandomResizedCrop,\n RandomPerspective,\n RandomRotation,\n Resize,\n RandAugment,\n ToImageTensor, \n ConvertImageDtype,\n ToTensor,\n)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-25T18:48:42.579123Z","iopub.execute_input":"2024-08-25T18:48:42.57959Z","iopub.status.idle":"2024-08-25T18:48:48.713532Z","shell.execute_reply.started":"2024-08-25T18:48:42.579563Z","shell.execute_reply":"2024-08-25T18:48:48.712418Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n warnings.warn(_BETA_TRANSFORMS_WARNING)\n/opt/conda/lib/python3.10/site-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n warnings.warn(_BETA_TRANSFORMS_WARNING)\n","output_type":"stream"}]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-08-25T18:48:56.422968Z","iopub.execute_input":"2024-08-25T18:48:56.423783Z","iopub.status.idle":"2024-08-25T18:48:56.430767Z","shell.execute_reply.started":"2024-08-25T18:48:56.42375Z","shell.execute_reply":"2024-08-25T18:48:56.429838Z"},"trusted":true},"execution_count":2,"outputs":[{"execution_count":2,"output_type":"execute_result","data":{"text/plain":"'cuda'"},"metadata":{}}]},{"cell_type":"code","source":"from transformers import AutoImageProcessor, ResNetForImageClassification\nprocessor = AutoImageProcessor.from_pretrained(\"microsoft/resnet-50\")\nmodel = ResNetForImageClassification.from_pretrained(\"microsoft/resnet-50\").to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T18:49:00.339793Z","iopub.execute_input":"2024-08-25T18:49:00.340565Z","iopub.status.idle":"2024-08-25T18:49:10.316387Z","shell.execute_reply.started":"2024-08-25T18:49:00.340535Z","shell.execute_reply":"2024-08-25T18:49:10.315365Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading (…)rocessor_config.json: 0%| | 0.00/266 [00:00 1\u001b[0m \u001b[43ma\u001b[49m\u001b[38;5;241m.\u001b[39mkeys()\n","\u001b[0;31mNameError\u001b[0m: name 'a' is not defined"],"ename":"NameError","evalue":"name 'a' is not defined","output_type":"error"}]},{"cell_type":"code","source":"model.eval()\nvdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\ntotal_loss=0\nn=0\nwith torch.no_grad():\n for a in vdl:\n labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n total_loss+=nn.CrossEntropyLoss()(labels,preds.logits.softmax(-1)).detach()\n n+=1\n vdl.set_description(f'average validation loss = {total_loss.item()/n}')","metadata":{"execution":{"iopub.status.busy":"2024-08-25T18:54:32.338052Z","iopub.execute_input":"2024-08-25T18:54:32.338397Z","iopub.status.idle":"2024-08-25T18:55:40.928652Z","shell.execute_reply.started":"2024-08-25T18:54:32.338371Z","shell.execute_reply":"2024-08-25T18:55:40.927507Z"},"trusted":true},"execution_count":29,"outputs":[{"name":"stderr","text":"average validation loss = 2.8704065595354353: 100%|██████████| 56/56 [01:08<00:00, 1.22s/it]\n","output_type":"stream"}]},{"cell_type":"code","source":"learnables=[]\nfor n,p in model.named_parameters():\n p.requires_grad = True\n if 'classifier' in n:\n learnables.append({'params':p,'lr':0.01})\n elif any(i in n for i in '345'):\n learnables.append({'params':p,'lr':0.001})\n else:\n p.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2024-08-25T18:55:58.248662Z","iopub.execute_input":"2024-08-25T18:55:58.249429Z","iopub.status.idle":"2024-08-25T18:55:58.256861Z","shell.execute_reply.started":"2024-08-25T18:55:58.249397Z","shell.execute_reply":"2024-08-25T18:55:58.255977Z"},"trusted":true},"execution_count":32,"outputs":[]},{"cell_type":"code","source":"optimizer=torch.optim.AdamW(learnables)\nEPOCHS=4\nloss_fn=nn.CrossEntropyLoss()\nfor epoch in range(EPOCHS):\n tdl=tqdm(DataLoader(dataset['train'],batch_size=64,num_workers=2,shuffle=True))\n model.train()\n for a in tdl:\n labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n loss=loss_fn(labels,preds.logits.softmax(-1))\n model.zero_grad(set_to_none=True)\n loss.backward()\n optimizer.step()\n tdl.set_description(f'train loss = {loss.item()}')\n model.eval()\n with torch.no_grad():\n total_loss=0\n n=0\n vdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\n for a in vdl:\n labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n total_loss+=loss_fn(labels,preds.logits.softmax(-1)).detach()\n n+=1\n vdl.set_description(f'average validation loss = {total_loss.item()/n}')\n \n ","metadata":{"execution":{"iopub.status.busy":"2024-08-25T18:56:01.821268Z","iopub.execute_input":"2024-08-25T18:56:01.821622Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stderr","text":"train loss = 2.5873677730560303: 100%|██████████| 317/317 [06:30<00:00, 1.23s/it]\naverage validation loss = 2.46666990007673: 100%|██████████| 56/56 [01:06<00:00, 1.18s/it] \ntrain loss = 2.5570883750915527: 92%|█████████▏| 292/317 [06:10<00:24, 1.02it/s]","output_type":"stream"}]},{"cell_type":"code","source":"_a=a['labels']\n_b=preds.logits.softmax(-1).argmax(-1)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:48:59.993962Z","iopub.execute_input":"2023-09-07T04:48:59.99493Z","iopub.status.idle":"2023-09-07T04:49:00.000855Z","shell.execute_reply.started":"2023-09-07T04:48:59.994894Z","shell.execute_reply":"2023-09-07T04:48:59.999866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(_a.detach().cpu().numpy(),_b.detach().cpu().numpy()))","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:53:11.321082Z","iopub.execute_input":"2023-09-07T04:53:11.322252Z","iopub.status.idle":"2023-09-07T04:53:11.338291Z","shell.execute_reply.started":"2023-09-07T04:53:11.322209Z","shell.execute_reply":"2023-09-07T04:53:11.337312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]#TODO\nvdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\nfor a in vdl:\n labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n total_loss+=loss_fn(labels,preds.logits.softmax(-1)).detach()\n n+=1\n vdl.set_description(f'average validation loss = {total_loss.item()/n}')\n\n","metadata":{},"execution_count":null,"outputs":[]}]}