{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import transformers\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport datasets\nfrom tqdm import tqdm\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader\nfrom datasets import load_dataset\nfrom torchvision.transforms.v2 import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor\nfrom torchvision.transforms.v2 import Compose, ColorJitter, ToTensor\nfrom torchvision.transforms.v2 import (\n    CenterCrop,\n    Compose,\n    Normalize,\n    RandomHorizontalFlip,\n    RandomResizedCrop,\n    RandomPerspective,\n    RandomRotation,\n    Resize,\n    RandAugment,\n    ToImageTensor, \n    ConvertImageDtype,\n    ToTensor,\n)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-07T06:21:49.471891Z","iopub.execute_input":"2023-09-07T06:21:49.472456Z","iopub.status.idle":"2023-09-07T06:21:56.108457Z","shell.execute_reply.started":"2023-09-07T06:21:49.472419Z","shell.execute_reply":"2023-09-07T06:21:56.106425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:21:56.110218Z","iopub.execute_input":"2023-09-07T06:21:56.111056Z","iopub.status.idle":"2023-09-07T06:21:56.119123Z","shell.execute_reply.started":"2023-09-07T06:21:56.111020Z","shell.execute_reply":"2023-09-07T06:21:56.118188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoImageProcessor, ResNetForImageClassification\nprocessor = AutoImageProcessor.from_pretrained(\"microsoft/resnet-50\")\nmodel = ResNetForImageClassification.from_pretrained(\"microsoft/resnet-50\").to(device)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:21:56.120818Z","iopub.execute_input":"2023-09-07T06:21:56.121497Z","iopub.status.idle":"2023-09-07T06:22:14.529206Z","shell.execute_reply.started":"2023-09-07T06:21:56.121467Z","shell.execute_reply":"2023-09-07T06:22:14.528130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = load_dataset(\"imagefolder\", data_dir=\"/kaggle/input/vehicle/train/train\")","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:22:14.535063Z","iopub.execute_input":"2023-09-07T06:22:14.537658Z","iopub.status.idle":"2023-09-07T06:24:42.246863Z","shell.execute_reply.started":"2023-09-07T06:22:14.537622Z","shell.execute_reply":"2023-09-07T06:24:42.245526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -n10 -r10\n(Compose(\n        [\n            RandomResizedCrop(processor.size['shortest_edge']),\n            RandAugment(),\n        ]\n    )(dataset['train'][0]['image']))","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:50:41.912184Z","iopub.execute_input":"2023-09-07T06:50:41.913169Z","iopub.status.idle":"2023-09-07T06:50:42.218734Z","shell.execute_reply.started":"2023-09-07T06:50:41.913126Z","shell.execute_reply":"2023-09-07T06:50:42.217627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%timeit -n10 -r10\n(nn.Sequential(\n    *[\n        \n        RandomResizedCrop(processor.size['shortest_edge']),\n        RandAugment(),\n        ConvertImageDtype(torch.float)\n    ]\n)(ToImageTensor()(dataset['train'][0]['image']).to('cuda')))","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:07:15.365707Z","iopub.execute_input":"2023-09-07T07:07:15.366074Z","iopub.status.idle":"2023-09-07T07:07:15.386094Z","shell.execute_reply.started":"2023-09-07T07:07:15.366044Z","shell.execute_reply":"2023-09-07T07:07:15.385188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = dataset['train'].train_test_split(test_size=0.15)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:24:42.258581Z","iopub.execute_input":"2023-09-07T06:24:42.258995Z","iopub.status.idle":"2023-09-07T06:24:43.083623Z","shell.execute_reply.started":"2023-09-07T06:24:42.258962Z","shell.execute_reply":"2023-09-07T06:24:43.082610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transforms(examples):\n    compose=Compose(\n        [\n            ToImageTensor(),\n            RandomResizedCrop(processor.size['shortest_edge']),\n            RandAugment(),\n            ConvertImageDtype()\n        ]\n    )\n    #precompose=\n    return {\n        'pixel_values':processor([compose(image) for image in examples[\"image\"]],return_tensors='pt')['pixel_values'],\n        'labels':examples['label']\n    }","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:09:23.220860Z","iopub.execute_input":"2023-09-07T07:09:23.221260Z","iopub.status.idle":"2023-09-07T07:09:23.228813Z","shell.execute_reply.started":"2023-09-07T07:09:23.221225Z","shell.execute_reply":"2023-09-07T07:09:23.227741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transforms(dataset['train'][:2])","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:08:14.361258Z","iopub.execute_input":"2023-09-07T07:08:14.361839Z","iopub.status.idle":"2023-09-07T07:08:14.369090Z","shell.execute_reply.started":"2023-09-07T07:08:14.361789Z","shell.execute_reply":"2023-09-07T07:08:14.368050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset['train'].set_transform(lambda x:x)\ndataset['test'].set_transform(lambda x:x)\n\ndataset['train'].set_transform(transforms)\ndataset['test'].set_transform(transforms)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:09:25.642325Z","iopub.execute_input":"2023-09-07T07:09:25.643051Z","iopub.status.idle":"2023-09-07T07:09:25.667207Z","shell.execute_reply.started":"2023-09-07T07:09:25.642994Z","shell.execute_reply":"2023-09-07T07:09:25.665768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.classifier[1]=nn.Linear(2048,17).to(device)\nmodel.config.num_labels=17\nmodel.num_labels=17","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:51:54.150698Z","iopub.execute_input":"2023-09-07T06:51:54.151072Z","iopub.status.idle":"2023-09-07T06:51:54.158628Z","shell.execute_reply.started":"2023-09-07T06:51:54.151042Z","shell.execute_reply":"2023-09-07T06:51:54.157557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.keys()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T06:52:44.461327Z","iopub.execute_input":"2023-09-07T06:52:44.462385Z","iopub.status.idle":"2023-09-07T06:52:44.472094Z","shell.execute_reply.started":"2023-09-07T06:52:44.462347Z","shell.execute_reply":"2023-09-07T06:52:44.471059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nvdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\ntotal_loss=0\nn=0\nwith torch.no_grad():\n    for a in vdl:\n        labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n        preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n        total_loss+=nn.CrossEntropyLoss()(labels,preds.logits.softmax(-1)).detach()\n        n+=1\n        vdl.set_description(f'average validation loss = {total_loss.item()/n}')","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:09:27.428862Z","iopub.execute_input":"2023-09-07T07:09:27.429232Z","iopub.status.idle":"2023-09-07T07:11:21.880940Z","shell.execute_reply.started":"2023-09-07T07:09:27.429200Z","shell.execute_reply":"2023-09-07T07:11:21.879306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learnables=[]\nfor n,p in model.named_parameters():\n    p.requires_grad = True\n    if 'classifier' in n:\n        learnables.append({'params':p,'lr':0.01})\n    elif any(i in n for i in '345'):\n        learnables.append({'params':p,'lr':0.001})\n    else:\n        p.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2023-09-07T03:58:34.312839Z","iopub.execute_input":"2023-09-07T03:58:34.313246Z","iopub.status.idle":"2023-09-07T03:58:34.322018Z","shell.execute_reply.started":"2023-09-07T03:58:34.313214Z","shell.execute_reply":"2023-09-07T03:58:34.320745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer=torch.optim.AdamW(learnables)\nEPOCHS=4\nloss_fn=nn.CrossEntropyLoss()\nfor epoch in range(EPOCHS):\n    tdl=tqdm(DataLoader(dataset['train'],batch_size=64,num_workers=2,shuffle=True))\n    model.train()\n    for a in tdl:\n        labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n        preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n        loss=loss_fn(labels,preds.logits.softmax(-1))\n        model.zero_grad(set_to_none=True)\n        loss.backward()\n        optimizer.step()\n        tdl.set_description(f'train loss = {loss.item()}')\n    model.eval()\n    with torch.no_grad():\n        total_loss=0\n        n=0\n        vdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\n        for a in vdl:\n            labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n            preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n            total_loss+=loss_fn(labels,preds.logits.softmax(-1)).detach()\n            n+=1\n            vdl.set_description(f'average validation loss = {total_loss.item()/n}')\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2023-09-07T03:59:30.793708Z","iopub.execute_input":"2023-09-07T03:59:30.794101Z","iopub.status.idle":"2023-09-07T04:47:48.808038Z","shell.execute_reply.started":"2023-09-07T03:59:30.794072Z","shell.execute_reply":"2023-09-07T04:47:48.806752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_a=a['labels']\n_b=preds.logits.softmax(-1).argmax(-1)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:48:59.993962Z","iopub.execute_input":"2023-09-07T04:48:59.994930Z","iopub.status.idle":"2023-09-07T04:49:00.000855Z","shell.execute_reply.started":"2023-09-07T04:48:59.994894Z","shell.execute_reply":"2023-09-07T04:48:59.999866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(_a.detach().cpu().numpy(),_b.detach().cpu().numpy()))","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:53:11.321082Z","iopub.execute_input":"2023-09-07T04:53:11.322252Z","iopub.status.idle":"2023-09-07T04:53:11.338291Z","shell.execute_reply.started":"2023-09-07T04:53:11.322209Z","shell.execute_reply":"2023-09-07T04:53:11.337312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]#TODO\nvdl=tqdm(DataLoader(dataset['test'],batch_size=64,num_workers=2,shuffle=True))\nfor a in vdl:\n    labels=nn.functional.one_hot(a['labels'].long(),17).float().to(device)\n    preds=model(**{k:a[k].to(device) for k in ('pixel_values','labels')})\n    total_loss+=loss_fn(labels,preds.logits.softmax(-1)).detach()\n    n+=1\n    vdl.set_description(f'average validation loss = {total_loss.item()/n}')\n\n","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}