{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"},{"sourceId":6348120,"sourceType":"datasetVersion","datasetId":3655658},{"sourceId":7556785,"sourceType":"datasetVersion","datasetId":4400759}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I previously performed FPGA aware Neural architecutral search and found different models with different latency through evolutionary algorithm.\n\nThe obtained network from search process will be now retrained to improve accuracy. Also these models wt were intialized with the help of OFA!","metadata":{"_uuid":"37fa20df-8fe2-4f9e-aa5a-3fb3236fe689","_cell_guid":"24bd6b67-4bf7-44c4-8b43-3bf1dd6032cb","trusted":true}},{"cell_type":"code","source":"import torch \nimport torchvision\nimport os\nimport torch.nn as nn\nfrom torchvision import transforms, datasets\nimport math\nimport time\nfrom tqdm import tqdm\nimport shutil","metadata":{"_uuid":"20994d4c-86b5-4bf0-97f3-b6b9a7491a9f","_cell_guid":"9682186b-4dd9-441b-adb9-6eed566f3b50","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T02:29:38.808265Z","iopub.execute_input":"2024-02-06T02:29:38.808985Z","iopub.status.idle":"2024-02-06T02:29:38.814127Z","shell.execute_reply.started":"2024-02-06T02:29:38.808949Z","shell.execute_reply":"2024-02-06T02:29:38.8129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cuda_available = torch.cuda.is_available()\nif cuda_available:\n    torch.backends.cudnn.enabled = True\n    torch.backends.cudnn.benchmark = True\n    print('Using GPU.')\nelse:\n    print('Using CPU.')","metadata":{"_uuid":"c43eb1c6-b89a-41d5-9bf1-273e0f778208","_cell_guid":"3f0a5eee-4ca9-4bfe-912b-fc0f52ec772b","collapsed":false,"execution":{"iopub.status.busy":"2024-02-05T11:57:41.246588Z","iopub.execute_input":"2024-02-05T11:57:41.247081Z","iopub.status.idle":"2024-02-05T11:57:41.279381Z","shell.execute_reply.started":"2024-02-05T11:57:41.247047Z","shell.execute_reply":"2024-02-05T11:57:41.278292Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbatch_size=32\n\n#I will use a susbset of imagenetval of 10k images \nif cuda_available:\n    # path to the ImageNet dataset\n    # link --> https://www.kaggle.com/datasets/titericz/imagenet1k-val\n    \n    imagenet_data_path = '/kaggle/input/imagenet1k-val/imagenet-val'\n\n    # if 'imagenet_data_path' is empty, download a subset of ImageNet containing 2000 images (~250M) for test\n    if not os.path.isdir(imagenet_data_path):\n        print('%s is empty. Download a subset of ImageNet for test.' % imagenet_data_path)\n\n    print('The ImageNet dataset files are ready.')\nelse:\n    print('Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.')\n    \n    \n  \nif cuda_available:\n    # The following function build the data transforms for test\n    def build_val_transform(size):\n        return transforms.Compose([\n            transforms.Resize(int(math.ceil(size / 0.875))),\n            transforms.CenterCrop(size),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            ),\n        ])\n    \n    val_data = datasets.ImageFolder(\n            root=os.path.join(imagenet_data_path),\n            transform=build_val_transform(224)\n        )\n    \n\n    val_loader = torch.utils.data.DataLoader(\n        val_data,\n        batch_size=batch_size,  \n        shuffle = True,\n        num_workers=4,  \n        pin_memory=True,\n        drop_last=False,\n    )\n    print('The ImageNet dataloader is ready. Size : {}'.format(len(val_loader)*batch_size))\nelse:\n    data_loader = None\n    print('Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.')","metadata":{"_uuid":"ec55af04-87e5-4479-8dbc-42a592c075ca","_cell_guid":"7819c409-ed1e-4fb7-93c3-9120866d1d66","collapsed":false,"execution":{"iopub.status.busy":"2024-02-05T11:57:44.469683Z","iopub.execute_input":"2024-02-05T11:57:44.470351Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_path = '/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train'\n\ntrain_data = datasets.ImageFolder(\n            root= train_path,\n            transform=build_val_transform(224)\n        )\n\ntrain_loader = torch.utils.data.DataLoader(\n        train_data,\n        batch_size=batch_size, \n        shuffle = True,\n        num_workers=4,  \n        pin_memory=True,\n        drop_last=False,\n    )\n\nprint('The ImageNet train set is ready. Size : {}'.format(len(train_loader)*batch_size))","metadata":{"_uuid":"90e0ebac-0aaa-4db9-bedc-59bddaa65f42","_cell_guid":"acb1a2cd-063d-4846-8789-a1944b28e6e0","collapsed":false,"execution":{"iopub.status.busy":"2024-02-05T02:23:09.242067Z","iopub.execute_input":"2024-02-05T02:23:09.24244Z","iopub.status.idle":"2024-02-05T02:47:25.930361Z","shell.execute_reply.started":"2024-02-05T02:23:09.24241Z","shell.execute_reply":"2024-02-05T02:47:25.929396Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = {}\ndataloaders['train'] = train_loader\ndataloaders['val'] = val_loader\n\ndataset_sizes = {'train': len(train_loader)*32,\n                'val': len(val_loader)*32}","metadata":{"_uuid":"fb3584e1-d8cc-4cf7-b3ca-ddec7d3c42c9","_cell_guid":"61be6f61-bc40-4e38-87c1-c60f943eb632","collapsed":false,"execution":{"iopub.status.busy":"2024-02-05T02:47:25.938136Z","iopub.execute_input":"2024-02-05T02:47:25.938373Z","iopub.status.idle":"2024-02-05T02:47:25.947764Z","shell.execute_reply.started":"2024-02-05T02:47:25.938352Z","shell.execute_reply":"2024-02-05T02:47:25.946895Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#Function for training the model, forware prop and backward prop \n\ndef train_model(model, criterion, optimizer, scheduler, num_epochs=5):\n    since = time.time()\n\n    #storing epoch data\n    epoch_data =     {\n        'epoch': [],\n        'train': {'loss': [], 'acc': []},\n        'val': {'loss': [], 'acc': [] }\n    }\n    \n    # Create a temporary directory in Kaggle's temp directory\n    tempdir = '/kaggle/working/temp'\n    os.makedirs(tempdir, exist_ok=True)\n    best_model_params_path = os.path.join(tempdir, 'best_model_params.pt')\n\n    torch.save(model.state_dict(), best_model_params_path)\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print('-' * 10)\n        epoch_data['epoch'].append(epoch+1)\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n            running_loss = 0.0\n            running_corrects = 0\n\n            \n\n            for inputs, labels in tqdm(dataloaders[phase], leave=False):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                optimizer.zero_grad()\n\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n            epoch_data[phase]['loss'].append(epoch_loss)\n            epoch_data[phase]['acc'].append(epoch_acc)\n\n            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                torch.save(model, best_model_params_path)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n    print(f'Best val Acc: {best_acc:4f}')\n\n    torch.load(best_model_params_path)\n\n    # Clean up the temporary directory\n    shutil.rmtree(tempdir)\n\n    return model, epoch_data","metadata":{"_uuid":"122dc352-bec3-4d79-be54-06dfec362441","_cell_guid":"0310d9dc-b729-4312-b42e-d16dc6b0ef6c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T02:30:48.766011Z","iopub.execute_input":"2024-02-06T02:30:48.766706Z","iopub.status.idle":"2024-02-06T02:30:48.783539Z","shell.execute_reply.started":"2024-02-06T02:30:48.766673Z","shell.execute_reply":"2024-02-06T02:30:48.782111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ofa ","metadata":{"execution":{"iopub.status.busy":"2024-02-06T02:42:16.573373Z","iopub.execute_input":"2024-02-06T02:42:16.573775Z","iopub.status.idle":"2024-02-06T02:42:31.465104Z","shell.execute_reply.started":"2024-02-06T02:42:16.573737Z","shell.execute_reply":"2024-02-06T02:42:31.463796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torch.load('/kaggle/input/searched-models-nas-fpga/models/moder_search_11.pth')","metadata":{"_uuid":"aaf96743-a919-43ed-8674-3e51dee4922e","_cell_guid":"73be0887-4e38-49c4-89bf-6f38a183b912","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T02:42:31.467141Z","iopub.execute_input":"2024-02-06T02:42:31.467483Z","iopub.status.idle":"2024-02-06T02:42:31.761569Z","shell.execute_reply.started":"2024-02-06T02:42:31.467454Z","shell.execute_reply":"2024-02-06T02:42:31.760734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if cuda_available else 'cpu'\n\nmodel = model.to(device)\ncriterion = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\noptimizer_ft = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.90)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)","metadata":{"_uuid":"bea12c60-ca18-4ead-8ebe-8dd147e72cea","_cell_guid":"9ead446b-60ed-4aae-9b72-ab7c8d002873","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T02:42:31.762808Z","iopub.execute_input":"2024-02-06T02:42:31.76318Z","iopub.status.idle":"2024-02-06T02:42:31.792816Z","shell.execute_reply.started":"2024-02-06T02:42:31.763147Z","shell.execute_reply":"2024-02-06T02:42:31.792104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\nmodel, epoch_data = train_model(model, criterion, optimizer_ft, exp_lr_scheduler,\n                       num_epochs=5)","metadata":{"_uuid":"be311f12-fc36-4ada-aa4b-220e19c37d3f","_cell_guid":"4baf48ad-b267-4732-814d-77711949212f","collapsed":false,"execution":{"iopub.status.busy":"2024-02-05T02:50:13.462945Z","iopub.execute_input":"2024-02-05T02:50:13.463496Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_train(epoch_data, title = ' '):\n    epochs = epoch_data['epoch']\n    train_loss = epoch_data['train']['loss']\n    val_loss = epoch_data['val']['loss']\n    train_acc = epoch_data['train']['acc']\n    val_acc = epoch_data['val']['acc']\n\n    # Plotting the training and validation loss\n    plt.figure(1)\n    plt.figure(figsize=(10,6))\n    plt.plot(epochs, train_loss, label='Training Loss', color='blue', linestyle='-', marker='o')\n    plt.plot(epochs, val_loss, label='Validation Loss', color='red', linestyle='--', marker='s')\n    plt.xlabel('Epoch', color='black')\n    plt.ylabel('Loss', color='black')\n    plt.title('Training and validation loss'+title, color='black')\n    plt.legend(loc='upper right', facecolor='white', framealpha=1)\n    plt.grid(color='gray', linestyle='--', linewidth=0.5)\n    plt.xticks(range(1,21))\n    plt.savefig('plot2.png')\n    # Plotting the training and validation accuracy\n    plt.figure(2)\n    plt.figure(figsize=(10,6))\n    plt.plot(epochs, train_acc, label='Training Accuracy', color='blue', linestyle='-', marker='^')\n    plt.plot(epochs, val_acc, label='Validation Accuracy', color='red', linestyle='--', marker='d')\n    plt.xlabel('Epoch', color='black')\n    plt.ylabel('Accuracy', color='black')\n    plt.title('Training and validation accuracy'+title, color='black')\n    plt.legend(loc='lower right', facecolor='white', framealpha=1)\n    plt.grid(color='gray', linestyle='--', linewidth=0.5)\n    plt.xticks(range(1,21))\n    \n    n = random.randint(0,100)\n    plt.savefig(f'plot{n}.png')\n\n    plt.show()","metadata":{"_uuid":"10da730e-0afb-410d-b697-39256262f42a","_cell_guid":"be50b2f5-99fc-4760-8ec3-0db9175ab4e3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_train(epoch_data, title = 'Retraining on Imagenet dataset (weight initialization from OFA)')","metadata":{"_uuid":"9833e603-6983-4d16-ac2b-40bcb17c29e3","_cell_guid":"e8c51de1-d59e-42d8-8678-c128bcebcb63","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, 'model_retrained_search_11.pth')","metadata":{"_uuid":"3e96f462-17ae-4f05-b3a6-38bb3b266494","_cell_guid":"cab47f0a-2bb4-4697-808b-3fadd4cb253d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(epoch_data)","metadata":{"_uuid":"97670473-b3a2-4a54-86a7-b74925f2217c","_cell_guid":"783f46c2-cb89-4496-a2ab-4a364848a359","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}