{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1195048,"sourceType":"datasetVersion","datasetId":680469},{"sourceId":3951115,"sourceType":"datasetVersion","datasetId":1027206}],"dockerImageVersionId":30528,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\nimport sys\nimport json\nimport math\nimport random\nimport cv2\ntimm_path = \"../input/timm-pytorch-image-models/pytorch-image-models-master\"\nsys.path.append(timm_path)\nimport timm\nfrom timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import model_selection\nfrom sklearn.metrics import mean_squared_error\nfrom tqdm.notebook import tqdm\nimport random\nimport glob\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch import optim\nfrom torchvision import transforms\nfrom transformers import  get_cosine_schedule_with_warmup\nimport warnings\nwarnings.filterwarnings('ignore')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"_uuid":"b91e6b8f-643e-4fc7-8592-fcd8d4c92e57","_cell_guid":"5fb38d86-6d5b-4eed-81f6-a767bae8fadf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:42.686767Z","iopub.execute_input":"2023-09-01T14:19:42.687165Z","iopub.status.idle":"2023-09-01T14:19:57.51244Z","shell.execute_reply.started":"2023-09-01T14:19:42.687129Z","shell.execute_reply":"2023-09-01T14:19:57.511433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\nset_seed(42)","metadata":{"_uuid":"771101f8-2e01-443c-9e3b-42b3b2f34d3d","_cell_guid":"f982d458-86f9-404f-93f0-47eb9d6fad17","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.515601Z","iopub.execute_input":"2023-09-01T14:19:57.516878Z","iopub.status.idle":"2023-09-01T14:19:57.529859Z","shell.execute_reply.started":"2023-09-01T14:19:57.516848Z","shell.execute_reply":"2023-09-01T14:19:57.528846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')","metadata":{"_uuid":"ac03c65c-5153-4a53-81ec-ee45ef34cd96","_cell_guid":"159ad0f8-13eb-4930-b41d-9614f02ac814","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.533391Z","iopub.execute_input":"2023-09-01T14:19:57.535743Z","iopub.status.idle":"2023-09-01T14:19:57.627312Z","shell.execute_reply.started":"2023-09-01T14:19:57.535715Z","shell.execute_reply":"2023-09-01T14:19:57.626338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"_uuid":"e2b1fdc7-99b5-41fa-adae-03e615c41f5e","_cell_guid":"b41ef9c1-6c8d-4d5a-9d85-636c28427cc5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.631799Z","iopub.execute_input":"2023-09-01T14:19:57.632106Z","iopub.status.idle":"2023-09-01T14:19:57.698701Z","shell.execute_reply.started":"2023-09-01T14:19:57.632066Z","shell.execute_reply":"2023-09-01T14:19:57.697787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['sex'] = df['sex'].map({'male': 1, 'female': 0})\ndf['sex'] = df['sex'].fillna(-1)\n\n\ndf['age_approx'] /= 90\ndf['age_approx'] = df['age_approx'].fillna(0)\n\n\ndf['n_images'] = df.patient_id.map(df.groupby(['patient_id']).image_name.count())\ndf.loc[df['patient_id'] == -1, 'n_images'] = 1","metadata":{"_uuid":"c7f49e0c-a9ad-4300-ad36-9e69278ad5be","_cell_guid":"4d1e9cfd-0900-48d4-bf1d-57345f376f24","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.70001Z","iopub.execute_input":"2023-09-01T14:19:57.700395Z","iopub.status.idle":"2023-09-01T14:19:57.738023Z","shell.execute_reply.started":"2023-09-01T14:19:57.700362Z","shell.execute_reply":"2023-09-01T14:19:57.737135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['path'] = [f\"/kaggle/input/siic-isic-224x224-images/train/{x}.png\" for x in df[\"image_name\"].values]\ndense_features = [\n    'sex', 'age_approx', 'n_images'\n]","metadata":{"_uuid":"6c7e7166-a1b7-4160-be57-12b8cfef5668","_cell_guid":"bd578b55-3213-44bb-bbed-db2b392c281a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.739234Z","iopub.execute_input":"2023-09-01T14:19:57.739975Z","iopub.status.idle":"2023-09-01T14:19:57.757494Z","shell.execute_reply.started":"2023-09-01T14:19:57.73994Z","shell.execute_reply":"2023-09-01T14:19:57.756541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"strat_kfold = model_selection.StratifiedGroupKFold(n_splits=5, random_state=42, shuffle=True)\n\n# Create an empty 'fold' column in the DataFrame\ndf['fold'] = -1  # Initialize with a default value\n\nfor i, (_, train_index) in enumerate(strat_kfold.split(df.index, y=df['target'], groups=df['patient_id'])):\n    df.loc[df.index[train_index], 'fold'] = i\n\ndf['fold'] = df['fold'].astype('int')","metadata":{"_uuid":"b56c4aba-da3b-4d90-be35-e9534c9ed5e7","_cell_guid":"1b7e842a-34ea-4ed1-bae0-3d48669b1cc3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:57.759707Z","iopub.execute_input":"2023-09-01T14:19:57.76005Z","iopub.status.idle":"2023-09-01T14:19:58.746658Z","shell.execute_reply.started":"2023-09-01T14:19:57.760019Z","shell.execute_reply":"2023-09-01T14:19:58.745636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"6090dcaf-106d-4db7-b691-f4c09fbb5aa5","_cell_guid":"15761693-b169-469a-a0d7-2b5a221310f7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:58.74796Z","iopub.execute_input":"2023-09-01T14:19:58.748503Z","iopub.status.idle":"2023-09-01T14:19:58.770198Z","shell.execute_reply.started":"2023-09-01T14:19:58.748469Z","shell.execute_reply":"2023-09-01T14:19:58.769335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sex.value_counts()","metadata":{"_uuid":"26e9b53e-b504-4eb2-abb3-f454da68e088","_cell_guid":"7f648bdd-f393-48b5-b374-56e133dfac82","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2023-09-01T14:19:58.771419Z","iopub.execute_input":"2023-09-01T14:19:58.771687Z","iopub.status.idle":"2023-09-01T14:19:58.782296Z","shell.execute_reply.started":"2023-09-01T14:19:58.771664Z","shell.execute_reply":"2023-09-01T14:19:58.78117Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.n_images = df.n_images/df.n_images.max()","metadata":{"_uuid":"f192f76a-fccc-47bb-9a82-29946d14e165","_cell_guid":"63de7154-af98-4224-951b-64dfd690144a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2023-09-01T14:19:58.788147Z","iopub.execute_input":"2023-09-01T14:19:58.788517Z","iopub.status.idle":"2023-09-01T14:19:58.794555Z","shell.execute_reply.started":"2023-09-01T14:19:58.788464Z","shell.execute_reply":"2023-09-01T14:19:58.793341Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_size = 224\ntrain_aug = A.Compose(\n    [   A.RandomResizedCrop(image_size,image_size,p= 0.8),\n        A.Resize(image_size,image_size,p=1.0),\n        A.HorizontalFlip(p=0.5),   \n        A.RandomBrightnessContrast(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=30, p=0.5),\n          \n   A.Normalize(mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD),\n        ToTensorV2()\n    ]\n)\nval_aug = A.Compose(\n    [ \n     A.Resize(image_size,image_size,p=1.0),\n        A.Normalize(mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD),\n        ToTensorV2()\n    ]\n)","metadata":{"_uuid":"61c22071-a00f-4950-997f-f706e0df1d58","_cell_guid":"69202e09-b32b-404a-bc14-570f4b3bcc38","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:58.796504Z","iopub.execute_input":"2023-09-01T14:19:58.796866Z","iopub.status.idle":"2023-09-01T14:19:58.808772Z","shell.execute_reply.started":"2023-09-01T14:19:58.796834Z","shell.execute_reply":"2023-09-01T14:19:58.807747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Op4bio_data(Dataset):\n    def __init__(self,df, augs):\n        self.df = df\n        self.augs = augs\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        img_src = self.df.loc[idx,'path']\n        image = cv2.imread(img_src)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        transformed = self.augs(image=image)\n        image = transformed['image']\n        \n        meta = self.df[dense_features].iloc[idx, :].values\n        \n        label  = self.df['target'][idx]\n        \n        return image , torch.FloatTensor(meta) , label","metadata":{"_uuid":"7af9feb6-8c01-4adf-9dd7-e491fa3f086a","_cell_guid":"5426d804-8686-43bf-9c1c-542f6bd13828","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:58.810434Z","iopub.execute_input":"2023-09-01T14:19:58.810855Z","iopub.status.idle":"2023-09-01T14:19:58.820671Z","shell.execute_reply.started":"2023-09-01T14:19:58.810823Z","shell.execute_reply":"2023-09-01T14:19:58.819715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t_data = Op4bio_data(df, augs = train_aug)","metadata":{"_uuid":"3d3e18c9-94f8-4837-903b-94ccdba4d5b4","_cell_guid":"e95b8b76-08dd-401d-ba32-59a774a261c4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2023-09-01T14:19:58.822214Z","iopub.execute_input":"2023-09-01T14:19:58.822627Z","iopub.status.idle":"2023-09-01T14:19:58.832405Z","shell.execute_reply.started":"2023-09-01T14:19:58.822594Z","shell.execute_reply":"2023-09-01T14:19:58.83148Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t_data[11]","metadata":{"_uuid":"bcc8aa4c-85fe-48af-8811-6b138d3f512b","_cell_guid":"06cfde9f-c5ce-4c9c-b709-4815af6ec882","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2023-09-01T14:19:58.833958Z","iopub.execute_input":"2023-09-01T14:19:58.83436Z","iopub.status.idle":"2023-09-01T14:19:59.00014Z","shell.execute_reply.started":"2023-09-01T14:19:58.834328Z","shell.execute_reply":"2023-09-01T14:19:58.999165Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,pretrained = True):\n        super().__init__()\n        self.backbone = timm.create_model('tf_efficientnet_b2_ns', pretrained=pretrained, num_classes=0, drop_rate=0.1,global_pool='',in_chans=3)\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.lin = nn.Linear(1408,64)\n        self.norm = nn.BatchNorm1d(1408)\n        self.do = nn.Dropout(p=0.4)\n    \n    def forward(self,image ):\n        image = self.backbone(image)     \n        image = self.pool(image)\n        image = image.view(image.shape[0], -1)\n        image = self.norm(image)\n        x = self.lin(image)\n        return x","metadata":{"_uuid":"1ca2589d-e587-4361-95b0-44ad073169c3","_cell_guid":"89854c6c-9376-4286-bb29-2c746092a67e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2023-09-01T14:19:59.002067Z","iopub.execute_input":"2023-09-01T14:19:59.002508Z","iopub.status.idle":"2023-09-01T14:19:59.010474Z","shell.execute_reply.started":"2023-09-01T14:19:59.002475Z","shell.execute_reply":"2023-09-01T14:19:59.009337Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"embedding_size = 64\n\nclass Lenet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn1 = nn.Conv2d(3, 6, kernel_size=5)\n        self.cnn2 = nn.Conv2d(6, 16, kernel_size=5)\n        \n        self.pool1 = nn.AvgPool2d(kernel_size=2, stride=2)\n        self.pool2 = nn.AvgPool2d(kernel_size=2, stride=2)\n        \n        self.relu1 = nn.ReLU()\n        self.relu2 = nn.ReLU()\n\n        # Calculate the input size for the fully connected layers\n        self.fc_input_size = 16 * 53 * 53  # This is for 224x224 input images\n        \n        self.fc1 = nn.Linear(self.fc_input_size, 120)\n        self.fc2 = nn.Linear(120, embedding_size)\n    \n    def forward(self, x):\n        x = self.relu1(self.cnn1(x))\n        x = self.pool1(x)\n        x = self.relu2(self.cnn2(x))\n        x = self.pool2(x)\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n\n        return x\n\n    \nclass LogisticRegression(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.layer = nn.Sequential(\n            nn.Linear(3, embedding_size),\n            nn.LeakyReLU(),\n            nn.BatchNorm1d(embedding_size),\n            nn.Linear( embedding_size,embedding_size),\n        )\n        \n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, x):\n        x = x.squeeze(1)\n        x = self.layer(x)\n        #x = self.sigmoid(x)\n        return x\n    \n    \nclass CLIPModel(nn.Module):\n    def __init__(self, model1, model2):\n        super(CLIPModel, self).__init__()\n        self.model1 = model1  # Lenet\n        self.model2 = model2  # Logistic\n        self.t = nn.Parameter(torch.Tensor([1]))  # Learnable parameter\n\n    def forward(self, x, y):\n        out1 = self.model1(x)\n        out2 = self.model2(y)\n        \n        #print(out1)\n        #print(out2)\n\n        # Apply L2 normalization\n        out1_norm = F.normalize(out1, p=2, dim=1)\n        out2_norm = F.normalize(out2, p=2, dim=1)\n        \n        out1_norm = out1_norm.to(out2_norm.dtype)  # Cast out1_norm to the same dtype as out2_norm\n\n\n        # Compute dot product\n        dot_product = torch.matmul(out1_norm, out2_norm.t())\n\n        # Apply exponential weighting\n        dot_product_weighted = torch.exp(self.t) * dot_product\n\n        return dot_product_weighted","metadata":{"_uuid":"f685dbf8-62d8-41ef-ad6c-6efb0f588b26","_cell_guid":"1b30eace-8c10-4246-9b77-55a1536edec4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.012026Z","iopub.execute_input":"2023-09-01T14:19:59.012654Z","iopub.status.idle":"2023-09-01T14:19:59.029009Z","shell.execute_reply.started":"2023-09-01T14:19:59.012622Z","shell.execute_reply":"2023-09-01T14:19:59.028151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"_uuid":"70148b6b-feea-4750-bf3a-d9b93dbe39fb","_cell_guid":"60e74245-6fe9-4c6c-9521-ff18bbfe8fd7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.031635Z","iopub.execute_input":"2023-09-01T14:19:59.031895Z","iopub.status.idle":"2023-09-01T14:19:59.044493Z","shell.execute_reply.started":"2023-09-01T14:19:59.031872Z","shell.execute_reply":"2023-09-01T14:19:59.04359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(train_loader,model,optimizer,criterion,e,epochs,device):\n    '''Trains the model for a single epoch and returns Loss,Accuracy, AUC for that epoch'''\n\n    losses = AverageMeter()\n    model.train()\n    global_step = 0\n    loop = tqdm(enumerate(train_loader),total = len(train_loader))\n    \n    for step,(image,tabular,l) in loop:\n        image = image.to(device)\n        tabular = tabular.to(device)\n        \n        batch_size = l.size(0)\n\n        output = model(image,tabular)\n        \n        #print(output)\n        \n        labels = torch.arange(batch_size) \n        labels = labels.to(device)\n        \n        loss1 = criterion(output, labels)\n        loss2 = criterion(output.T, labels)\n        loss = (loss1 + loss2) / 2.0\n        \n        losses.update(loss.item(), batch_size)\n\n        loss.backward()\n         # Clip gradients to avoid gradient explosion\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Adjust max_norm value as needed\n        \n        \n        optimizer.step()\n        optimizer.zero_grad()\n\n        global_step += 1\n\n        loop.set_description(f\"Epoch {e+1}/{epochs}\")\n        loop.set_postfix(model_loss = loss.item() ,stage = 'train')        \n\n    return losses.avg","metadata":{"_uuid":"5a52cbe6-551b-419c-8b86-e525d24dd5b8","_cell_guid":"d223f71e-2d20-4a41-9a4e-5271619e9037","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.046022Z","iopub.execute_input":"2023-09-01T14:19:59.046652Z","iopub.status.idle":"2023-09-01T14:19:59.063931Z","shell.execute_reply.started":"2023-09-01T14:19:59.046621Z","shell.execute_reply":"2023-09-01T14:19:59.062956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def val_one_epoch(loader,model,criterion,device):\n    '''Validates the model for a single epoch and returns Loss,Accuracy, AUC for that epoch'''\n    losses = AverageMeter()\n    model.eval()\n    global_step = 0\n    loop = tqdm(enumerate(loader),total = len(loader))\n    \n    for step,(image,tabular,l) in loop:\n        image = image.to(device)\n        tabular = tabular.to(device)\n        batch_size = l.size(0)\n        \n        labels = torch.arange(batch_size) \n        labels = labels.to(device)\n        \n        with torch.no_grad():\n            output = model(image,tabular)\n\n        loss1 = criterion(output, labels)\n        loss2 = criterion(output.T, labels)\n        loss = (loss1 + loss2) / 2.0        \n        losses.update(loss.item(), batch_size)\n        loop.set_postfix(model_loss = loss.item() ,stage = 'val')\n        global_step += 1\n\n    return losses.avg","metadata":{"_uuid":"e66e9532-bc54-4629-8120-fd0c20473a0e","_cell_guid":"1cb4bf1a-e847-4629-ab0d-e1eaa177e697","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.066206Z","iopub.execute_input":"2023-09-01T14:19:59.06729Z","iopub.status.idle":"2023-09-01T14:19:59.079544Z","shell.execute_reply.started":"2023-09-01T14:19:59.067255Z","shell.execute_reply":"2023-09-01T14:19:59.078662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fit(t_loader ,v_loader, model, OUTPUT_DIR,device,optimizer):\n    \n    T_LOSS1 = []\n    V_LOSS1 = []\n    model.to(device)\n    #model.to(device)\n    \n    criterion = nn.CrossEntropyLoss() # Loss function\n    optimizer = optimizer\n    \n    epochs = 25\n\n    loop = range(epochs)\n    for e in loop:\n      \n        loss = train_one_epoch(t_loader,model,optimizer,criterion,e,epochs,device)\n        \n        print(f'For epoch {e+1}/{epochs}')\n        print(f'average model train_loss {loss}')\n        \n        T_LOSS1.append(loss)\n\n        val_loss = val_one_epoch(v_loader,model,criterion,device)\n        \n        print(f'average model val_loss {val_loss}')\n        \n        V_LOSS1.append(val_loss)\n\n        torch.save(model.state_dict(),OUTPUT_DIR+ f'model_val_loss {val_loss}.pth')\n\n    return T_LOSS1,V_LOSS1","metadata":{"_uuid":"80e00b9d-a30c-4453-8297-0b0138da87db","_cell_guid":"986a6f75-7213-4357-940d-f6b72c91a9a5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.082285Z","iopub.execute_input":"2023-09-01T14:19:59.083416Z","iopub.status.idle":"2023-09-01T14:19:59.097389Z","shell.execute_reply.started":"2023-09-01T14:19:59.083381Z","shell.execute_reply":"2023-09-01T14:19:59.096521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data= df[df.fold != 0]\nval_data  = df[df.fold == 0]\n    \nt_data = Op4bio_data(train_data.reset_index(drop=True) , augs = train_aug)\nv_data = Op4bio_data(val_data.reset_index(drop=True) , augs = val_aug)\n\n\nt_loader = DataLoader(t_data, shuffle=True,\n                        num_workers=2,\n                        batch_size=16*6,drop_last =True)\n\nv_loader = DataLoader(v_data, shuffle=False,\n                        num_workers=2,\n                        batch_size=16*8,drop_last =False)","metadata":{"_uuid":"874f004f-82fe-44b5-9490-68eaee7a982f","_cell_guid":"0129f9af-268f-46dc-8fa8-463474541e25","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.098864Z","iopub.execute_input":"2023-09-01T14:19:59.099218Z","iopub.status.idle":"2023-09-01T14:19:59.117297Z","shell.execute_reply.started":"2023-09-01T14:19:59.099188Z","shell.execute_reply":"2023-09-01T14:19:59.116172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","metadata":{"_uuid":"2b07f14d-e482-4207-8ab6-90789174019d","_cell_guid":"546516a3-10a6-44b6-bef2-644b51e9ecc1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.118834Z","iopub.execute_input":"2023-09-01T14:19:59.119825Z","iopub.status.idle":"2023-09-01T14:19:59.124244Z","shell.execute_reply.started":"2023-09-01T14:19:59.119799Z","shell.execute_reply":"2023-09-01T14:19:59.123324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model1 =  Model()\nmodel2 =  LogisticRegression()","metadata":{"_uuid":"d7bf1a4c-75d7-4a10-87ec-d85004407ef2","_cell_guid":"d04f575c-8480-4499-a8d3-38de7dff1d9c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:19:59.12575Z","iopub.execute_input":"2023-09-01T14:19:59.126394Z","iopub.status.idle":"2023-09-01T14:20:00.028768Z","shell.execute_reply.started":"2023-09-01T14:19:59.126363Z","shell.execute_reply":"2023-09-01T14:20:00.027837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_model = CLIPModel(model1, model2)\noptimizer = optim.AdamW(combined_model.parameters(), lr=1e-6 , weight_decay = 1e-5 ) \nT_LOSS1, V_LOSS1= fit(t_loader ,v_loader, combined_model, OUTPUT_DIR,device,optimizer )","metadata":{"_uuid":"16352c1a-b731-4d39-b78b-c93f186c9c0e","_cell_guid":"d201eeab-f68b-40d5-aa8f-233ee4af7520","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T14:20:00.030207Z","iopub.execute_input":"2023-09-01T14:20:00.030643Z","iopub.status.idle":"2023-09-01T15:27:51.503579Z","shell.execute_reply.started":"2023-09-01T14:20:00.03061Z","shell.execute_reply":"2023-09-01T15:27:51.502007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Generate x-axis values\nepochs = len(T_LOSS1)\nx = list(range(1, epochs + 1))\n\n# Plot training loss\nplt.plot(x, T_LOSS1, label='Training Loss')\n\n# Plot validation loss\nplt.plot(x, V_LOSS1, label='Validation Loss')\n\n# Set plot labels and title\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training and Validation Loss')\nplt.legend()\n\n# Show the plot\nplt.show()","metadata":{"_uuid":"fc340d20-4e2d-4721-ac29-f82c3e2aa860","_cell_guid":"1237cc9e-e53a-45c9-bf81-73252106f268","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-01T15:27:51.505205Z","iopub.status.idle":"2023-09-01T15:27:51.505978Z","shell.execute_reply.started":"2023-09-01T15:27:51.505733Z","shell.execute_reply":"2023-09-01T15:27:51.505757Z"}},"outputs":[],"execution_count":null}]}