{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-29T15:51:26.093672Z","iopub.execute_input":"2023-10-29T15:51:26.093974Z","iopub.status.idle":"2023-10-29T15:51:26.099439Z","shell.execute_reply.started":"2023-10-29T15:51:26.093947Z","shell.execute_reply":"2023-10-29T15:51:26.098476Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"pip install open-clip-torch","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:29:21.872022Z","iopub.execute_input":"2023-11-02T11:29:21.872341Z","iopub.status.idle":"2023-11-02T11:29:36.85654Z","shell.execute_reply.started":"2023-11-02T11:29:21.872313Z","shell.execute_reply":"2023-11-02T11:29:36.855449Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting open-clip-torch\n  Downloading open_clip_torch-2.23.0-py3-none-any.whl (1.5 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n\u001b[?25hRequirement already satisfied: torch>=1.9.0 in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (2.0.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (0.15.1)\nRequirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (2023.6.3)\nCollecting ftfy (from open-clip-torch)\n  Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (4.66.1)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (0.16.4)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (0.1.99)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (3.20.3)\nRequirement already satisfied: timm in /opt/conda/lib/python3.10/site-packages (from open-clip-torch) (0.9.7)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open-clip-torch) (3.12.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open-clip-torch) (4.6.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open-clip-torch) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open-clip-torch) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open-clip-torch) (3.1.2)\nRequirement already satisfied: wcwidth>=0.2.5 in /opt/conda/lib/python3.10/site-packages (from ftfy->open-clip-torch) (0.2.6)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open-clip-torch) (2023.9.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open-clip-torch) (2.31.0)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open-clip-torch) (6.0)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open-clip-torch) (21.3)\nRequirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from timm->open-clip-torch) (0.3.3)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from torchvision->open-clip-torch) (1.23.5)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.10/site-packages (from torchvision->open-clip-torch) (9.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface-hub->open-clip-torch) (3.0.9)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.9.0->open-clip-torch) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open-clip-torch) (3.1.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open-clip-torch) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open-clip-torch) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open-clip-torch) (2023.7.22)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.9.0->open-clip-torch) (1.3.0)\nInstalling collected packages: ftfy, open-clip-torch\nSuccessfully installed ftfy-6.1.1 open-clip-torch-2.23.0\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"import open_clip","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:29:39.49875Z","iopub.execute_input":"2023-11-02T11:29:39.499106Z","iopub.status.idle":"2023-11-02T11:29:49.925556Z","shell.execute_reply.started":"2023-11-02T11:29:39.499073Z","shell.execute_reply":"2023-11-02T11:29:49.924699Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"import glob\nimport sys\nimport os\nimport time\nimport random\nimport math\n\n# DATALOADER\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport albumentations as A\nimport torchvision.transforms as T\nfrom PIL import Image\nimport pandas as pd\n\n# BUILDING MODEL\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models  # Import ResNet-50\n\n# TRAINING\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:31:54.219092Z","iopub.execute_input":"2023-11-02T11:31:54.219772Z","iopub.status.idle":"2023-11-02T11:31:57.34606Z","shell.execute_reply.started":"2023-11-02T11:31:54.21974Z","shell.execute_reply":"2023-11-02T11:31:57.34531Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n","output_type":"stream"}]},{"cell_type":"code","source":"import pandas as pd\nfrom IPython.display import display\nfrom PIL import Image\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport random\nimport seaborn as sns\nfrom keras.preprocessing import image\nfrom keras.models import Model, load_model\nimport tensorflow as tf\nfrom tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input\nimport matplotlib.image as mpimg\nimport pickle\nimport pathlib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import transforms, models\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom transformers import get_cosine_schedule_with_warmup\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:31:57.34744Z","iopub.execute_input":"2023-11-02T11:31:57.347866Z","iopub.status.idle":"2023-11-02T11:32:09.796958Z","shell.execute_reply.started":"2023-11-02T11:31:57.347839Z","shell.execute_reply":"2023-11-02T11:32:09.795972Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"pip install faiss-cpu\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:32:30.505916Z","iopub.execute_input":"2023-11-02T11:32:30.506712Z","iopub.status.idle":"2023-11-02T11:32:43.719675Z","shell.execute_reply.started":"2023-11-02T11:32:30.506674Z","shell.execute_reply":"2023-11-02T11:32:43.718326Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Collecting faiss-cpu\n  Downloading faiss_cpu-1.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.6 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m67.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: faiss-cpu\nSuccessfully installed faiss-cpu-1.7.4\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"pip install timm\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:38:38.688854Z","iopub.execute_input":"2023-11-02T11:38:38.689756Z","iopub.status.idle":"2023-11-02T11:38:50.849088Z","shell.execute_reply.started":"2023-11-02T11:38:38.689708Z","shell.execute_reply":"2023-11-02T11:38:50.847889Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Requirement already satisfied: timm in /opt/conda/lib/python3.10/site-packages (0.9.7)\nRequirement already satisfied: torch>=1.7 in /opt/conda/lib/python3.10/site-packages (from timm) (2.0.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.10/site-packages (from timm) (0.15.1)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from timm) (6.0)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from timm) (0.16.4)\nRequirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from timm) (0.3.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.7->timm) (3.12.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.7->timm) (4.6.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.7->timm) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.7->timm) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.7->timm) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->timm) (2023.9.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->timm) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->timm) (4.66.1)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->timm) (21.3)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from torchvision->timm) (1.23.5)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.10/site-packages (from torchvision->timm) (9.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface-hub->timm) (3.0.9)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.7->timm) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->timm) (3.1.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->timm) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->timm) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->timm) (2023.7.22)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.7->timm) (1.3.0)\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"pip install transformers\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:38:53.968329Z","iopub.execute_input":"2023-11-02T11:38:53.969115Z","iopub.status.idle":"2023-11-02T11:39:06.061992Z","shell.execute_reply.started":"2023-11-02T11:38:53.969076Z","shell.execute_reply":"2023-11-02T11:39:06.060571Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.33.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.2)\nRequirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.16.4)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.6.3)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.31.0)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.3)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.3.3)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.66.1)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.9.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (4.6.3)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.0.9)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.1.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2023.7.22)\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"pip install pytorch-metric-learning\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:39:06.064638Z","iopub.execute_input":"2023-11-02T11:39:06.064943Z","iopub.status.idle":"2023-11-02T11:39:18.35931Z","shell.execute_reply.started":"2023-11-02T11:39:06.064911Z","shell.execute_reply":"2023-11-02T11:39:18.358091Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"Collecting pytorch-metric-learning\n  Downloading pytorch_metric_learning-2.3.0-py3-none-any.whl (115 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (1.23.5)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (1.2.2)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (4.66.1)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (2.0.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.12.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (4.6.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.1.2)\nRequirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (1.11.2)\nRequirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (1.3.2)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (3.1.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->pytorch-metric-learning) (2.1.3)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->pytorch-metric-learning) (1.3.0)\nInstalling collected packages: pytorch-metric-learning\nSuccessfully installed pytorch-metric-learning-2.3.0\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"import torchvision\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:39:18.361174Z","iopub.execute_input":"2023-11-02T11:39:18.361645Z","iopub.status.idle":"2023-11-02T11:39:18.36677Z","shell.execute_reply.started":"2023-11-02T11:39:18.361598Z","shell.execute_reply":"2023-11-02T11:39:18.365915Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"torch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:39:18.369184Z","iopub.execute_input":"2023-11-02T11:39:18.369518Z","iopub.status.idle":"2023-11-02T11:39:18.465181Z","shell.execute_reply.started":"2023-11-02T11:39:18.369493Z","shell.execute_reply":"2023-11-02T11:39:18.464226Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"import glob\nimport sys\nimport os\nimport time\nimport random\nimport math\n\n# DATALOADER\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport albumentations as A\nimport torchvision.transforms as T\nfrom PIL import Image\nimport pandas as pd\n\n# BUILDING MODEL\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# TRAINING\nfrom torch.utils.data import DataLoader, Dataset\nimport faiss\nfrom tqdm import tqdm_notebook as tqdm\n\n# OTHER STUFF\nimport timm\nfrom transformers import (get_linear_schedule_with_warmup, \n                          get_constant_schedule,\n                          get_cosine_schedule_with_warmup, \n                          get_cosine_with_hard_restarts_schedule_with_warmup,\n                          get_constant_schedule_with_warmup)\nimport gc\nimport transformers\nfrom transformers import CLIPProcessor, CLIPVisionModel,  CLIPVisionConfig\nfrom pytorch_metric_learning import losses\nimport open_clip\n\n\n\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:39:18.466608Z","iopub.execute_input":"2023-11-02T11:39:18.467296Z","iopub.status.idle":"2023-11-02T11:39:19.29474Z","shell.execute_reply.started":"2023-11-02T11:39:18.467242Z","shell.execute_reply":"2023-11-02T11:39:19.293955Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport albumentations as A\nimport torchvision.transforms as T\n\n\n!pip install open_clip_torch\nimport open_clip\nimport cv2\nfrom PIL import Image\nimport yaml\nimport math\n\nimport os #to import custom models\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import DataLoader, Dataset, ConcatDataset\n!pip install faiss-cpu\n!pip install faiss-gpu\nimport faiss\n#from tqdm import tqdm\nfrom tqdm import tqdm_notebook as tqdm\n#!pip install loguru\n#from loguru import logger\nfrom sklearn.preprocessing import normalize\nfrom torch.utils.data import ConcatDataset\n\nimport timm\nimport glob\nfrom transformers import (get_linear_schedule_with_warmup, \n                          get_constant_schedule,\n                          get_cosine_schedule_with_warmup, \n                          get_cosine_with_hard_restarts_schedule_with_warmup,\n                          get_constant_schedule_with_warmup)\nimport gc\nimport transformers\nfrom transformers import CLIPProcessor, CLIPVisionModel,  CLIPVisionConfig\n\n!pip install pytorch-metric-learning\nfrom pytorch_metric_learning import losses\nfrom sklearn.model_selection import train_test_split\n\nimport copy\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport time\n#from torchvision import transforms\nimport random","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:39:19.295848Z","iopub.execute_input":"2023-11-02T11:39:19.296125Z","iopub.status.idle":"2023-11-02T11:40:10.225981Z","shell.execute_reply.started":"2023-11-02T11:39:19.296101Z","shell.execute_reply":"2023-11-02T11:40:10.225006Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"Requirement already satisfied: open_clip_torch in /opt/conda/lib/python3.10/site-packages (2.23.0)\nRequirement already satisfied: torch>=1.9.0 in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (2.0.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (0.15.1)\nRequirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (2023.6.3)\nRequirement already satisfied: ftfy in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (6.1.1)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (4.66.1)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (0.16.4)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (0.1.99)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (3.20.3)\nRequirement already satisfied: timm in /opt/conda/lib/python3.10/site-packages (from open_clip_torch) (0.9.7)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch) (3.12.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch) (4.6.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.9.0->open_clip_torch) (3.1.2)\nRequirement already satisfied: wcwidth>=0.2.5 in /opt/conda/lib/python3.10/site-packages (from ftfy->open_clip_torch) (0.2.6)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch) (2023.9.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch) (2.31.0)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch) (6.0)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->open_clip_torch) (21.3)\nRequirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from timm->open_clip_torch) (0.3.3)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from torchvision->open_clip_torch) (1.23.5)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.10/site-packages (from torchvision->open_clip_torch) (9.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface-hub->open_clip_torch) (3.0.9)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.9.0->open_clip_torch) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open_clip_torch) (3.1.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open_clip_torch) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open_clip_torch) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->open_clip_torch) (2023.7.22)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.9.0->open_clip_torch) (1.3.0)\nRequirement already satisfied: faiss-cpu in /opt/conda/lib/python3.10/site-packages (1.7.4)\nCollecting faiss-gpu\n  Downloading faiss_gpu-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (85.5 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: faiss-gpu\nSuccessfully installed faiss-gpu-1.7.2\nRequirement already satisfied: pytorch-metric-learning in /opt/conda/lib/python3.10/site-packages (2.3.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (1.23.5)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (1.2.2)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (4.66.1)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from pytorch-metric-learning) (2.0.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.12.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (4.6.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->pytorch-metric-learning) (3.1.2)\nRequirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (1.11.2)\nRequirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (1.3.2)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->pytorch-metric-learning) (3.1.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->pytorch-metric-learning) (2.1.3)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->pytorch-metric-learning) (1.3.0)\n","output_type":"stream"}]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\nimport numpy as np\nimport os\nimport faiss\nimport copy\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom PIL import Image\nfrom torchvision import transforms\nimport cv2\nimport time\n\nclass ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, \n                 m=0.50, easy_margin=False, ls_eps=0.0, device=torch.device('cuda')):\n        super(ArcMarginProduct, self).__init__()\n        self.device = device\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=self.device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\n\nclass DenseCrossEntropy(nn.Module):\n    def forward(self, x, target):\n        x = x.float()\n        target = target.float()\n        logprobs = torch.nn.functional.log_softmax(x, dim=-1)\n\n        loss = -logprobs * target\n        loss = loss.sum(-1)\n        return loss.mean()\n    \nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.gamma = gamma\n\n    def forward(self, x, target):\n        x = x.float()\n        target = target.float()\n        probs = torch.nn.functional.softmax(x, dim=-1)\n        logprobs = torch.log(probs)\n\n        loss = -logprobs * target * (1 - probs) ** self.gamma\n        loss = loss.sum(-1)\n        return loss.mean()\n\nclass ArcMarginProduct_subcenter(nn.Module):\n    def __init__(self, in_features, out_features, k=3):\n        super().__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(out_features*k, in_features))\n        self.reset_parameters()\n        self.k = k\n        self.out_features = out_features\n        \n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n        \n    def forward(self, features):\n        cosine_all = F.linear(F.normalize(features), F.normalize(self.weight))\n        cosine_all = cosine_all.view(-1, self.out_features, self.k)\n        cosine, _ = torch.max(cosine_all, dim=2)\n        return cosine   \n\nclass ArcFaceLossAdaptiveMargin(nn.modules.Module):\n    def __init__(self, margins, s=30.0, crit='ce'):\n        super().__init__()\n        if crit == 'ce':\n            self.crit = DenseCrossEntropy()\n        else:\n            self.crit = FocalLoss()\n        self.s = s\n        self.margins = margins\n            \n    def forward(self, logits, labels, out_dim):\n        ms = []\n        ms = self.margins[labels.cpu().numpy()]\n        cos_m = torch.from_numpy(np.cos(ms)).float().cuda()\n        sin_m = torch.from_numpy(np.sin(ms)).float().cuda()\n        th = torch.from_numpy(np.cos(math.pi - ms)).float().cuda()\n        mm = torch.from_numpy(np.sin(math.pi - ms) * ms).float().cuda()\n        labels = F.one_hot(labels, out_dim).float()\n        logits = logits.float()\n        cosine = logits\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * cos_m.view(-1,1) - sine * sin_m.view(-1,1)\n        phi = torch.where(cosine > th.view(-1,1), phi, cosine - mm.view(-1,1))\n        output = (labels * phi) + ((1.0 - labels) * cosine)\n        output *= self.s\n        loss = self.crit(output, labels)\n        return loss     \n\ndef set_seed(seed):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n\ndef get_similiarity_hnsw(embeddings_gallery, emmbeddings_query, k):\n    # this is guy is really fast\n    print('Processing indices...')\n\n    s = time.time()\n    index = faiss.IndexHNSWFlat(embeddings_gallery.shape[1], 32)\n    index.add(embeddings_gallery)\n\n    scores, indices = index.search(emmbeddings_query, k) \n    e = time.time()\n\n    print(f'Finished processing indices, took {e - s}s')\n    return scores, indices\n\ndef get_similiarity_l2(embeddings_gallery, emmbeddings_query, k):\n    print('Processing indices...')\n\n    s = time.time()\n    index = faiss.IndexFlatL2(embeddings_gallery.shape[1])\n    index.add(embeddings_gallery)\n\n    scores, indices = index.search(emmbeddings_query, k) \n    e = time.time()\n\n    print(f'Finished processing indices, took {e - s}s')\n    return scores, indices\n\n\ndef get_similiarity_IP(embeddings_gallery, emmbeddings_query, k):\n    print('Processing indices...')\n\n    s = time.time()\n    index = faiss.IndexFlatIP(embeddings_gallery.shape[1])\n    index.add(embeddings_gallery)\n\n    scores, indices = index.search(emmbeddings_query, k) \n    e = time.time()\n\n    print(f'Finished processing indices, took {e - s}s')\n    return scores, indices\n\ndef get_similiarity(embeddings, k):\n    print('Processing indices...')\n\n    index = faiss.IndexFlatL2(embeddings.shape[1])\n\n    res = faiss.StandardGpuResources()\n\n    index = faiss.index_cpu_to_gpu(res, 0, index)\n\n    index.add(embeddings)\n\n    scores, indices = index.search(embeddings, k) \n    print('Finished processing indices')\n\n    return scores, indices\n\ndef map_per_image(label, predictions, k=5): \n    try:\n        return 1 / (predictions[:k].index(label) + 1)\n    except ValueError:\n        return 0.0\n\ndef map_per_set(labels, predictions, k=5):\n    return np.mean([map_per_image(l, p, k) for l,p in zip(labels, predictions)])\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self, window_size=None):\n        self.length = 0\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        self.window_size = window_size\n\n    def reset(self):\n        self.length = 0\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        if self.window_size and (self.count >= self.window_size):\n            self.reset()\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef get_lr_groups(param_groups):\n        groups = sorted(set([param_g['lr'] for param_g in param_groups]))\n        groups = [\"{:2e}\".format(group) for group in groups]\n        return groups\n\ndef convert_indices_to_labels(indices, labels):\n    indices_copy = copy.deepcopy(indices)\n    for row in indices_copy:\n        for j in range(len(row)):\n            row[j] = labels[row[j]]\n    return indices_copy\n\nclass Multisample_Dropout(nn.Module):\n    def __init__(self, dropout_rate=0.1):\n        super(Multisample_Dropout, self).__init__()\n        self.dropout = nn.Dropout(dropout_rate)\n        self.dropouts = nn.ModuleList([nn.Dropout((i+1)*.1) for i in range(5)])\n        \n    def forward(self, x, module):\n        x = self.dropout(x)\n        return torch.mean(torch.stack([module(dropout(x)) for dropout in self.dropouts],dim=0),dim=0) \n\ndef transforms_auto_augment(image_path, image_size):\n    image = Image.open(image_path).convert('RGB')\n    train_transforms = transforms.Compose([transforms.AutoAugment(transforms.AutoAugmentPolicy.IMAGENET), transforms.PILToTensor()])\n    return train_transforms(image)\n\ndef transforms_cutout(image_path, image_size):\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.uint8)\n    train_transforms = A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.ImageCompression(quality_lower=99, quality_upper=100),\n            A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n            A.Resize(image_size, image_size),\n            A.Cutout(max_h_size=int(image_size * 0.4), max_w_size=int(image_size * 0.4), num_holes=1, p=0.5),\n            ToTensorV2(),\n        ])\n    return train_transforms(image=image)['image']\n\ndef transforms_happy_whale(image_path, image_size):\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.uint8)\n    aug8p3 = A.OneOf([\n            A.Sharpen(p=0.3),\n            A.ToGray(p=0.3),\n            A.CLAHE(p=0.3),\n        ], p=0.5)\n\n    train_transforms = A.Compose([\n            A.ShiftScaleRotate(rotate_limit=15, scale_limit=0.1, border_mode=cv2.BORDER_REFLECT, p=0.5),\n            A.Resize(image_size, image_size),\n            aug8p3,\n            A.HorizontalFlip(p=0.5),\n            A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n            ToTensorV2(),\n        ])\n    return train_transforms(image=image)['image']\n\ndef transforms_valid(image_path, image_size):\n    image = Image.open(image_path).convert('RGB')\n    valid_transforms = transforms.Compose([transforms.PILToTensor()]) \n    return valid_transforms(image)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:40:10.227793Z","iopub.execute_input":"2023-11-02T11:40:10.228175Z","iopub.status.idle":"2023-11-02T11:40:10.400647Z","shell.execute_reply.started":"2023-11-02T11:40:10.228135Z","shell.execute_reply":"2023-11-02T11:40:10.399827Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:40:10.402033Z","iopub.execute_input":"2023-11-02T11:40:10.402494Z","iopub.status.idle":"2023-11-02T11:40:10.503986Z","shell.execute_reply.started":"2023-11-02T11:40:10.402459Z","shell.execute_reply":"2023-11-02T11:40:10.503062Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"class CFG:\n#     model_name = 'ViT-H-14' \n#     model_data = 'laion2b_s32b_b79k'\n    model_name = 'ViT-B-32-quickgelu' \n    model_data = 'laion400m_e32'\n    samples_per_class = 50\n    n_classes = 0\n    min_samples = 4\n    image_size = 224 \n    hidden_layer = 512  #1024\n    seed = 5\n    workers = 12\n    train_batch_size = 8\n    valid_batch_size = 32 \n    emb_size = 512\n    vit_bb_lr = {'10': 1.25e-6, '20': 2.5e-6, '26': 5e-6, '32': 10e-6} \n    vit_bb_wd = 1e-3\n    hd_lr = 3e-4\n    hd_wd = 1e-5\n    autocast = True\n    n_warmup_steps = 1000\n    n_epochs = 10\n    device = torch.device('cuda')\n    s=30.\n    m=.45\n    m_min=.05\n    acc_steps = 4\n    global_step = 0\n    reduce_lr = 0.1\n    crit = 'ce'\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:40:10.505238Z","iopub.execute_input":"2023-11-02T11:40:10.505584Z","iopub.status.idle":"2023-11-02T11:40:10.609779Z","shell.execute_reply.started":"2023-11-02T11:40:10.505547Z","shell.execute_reply":"2023-11-02T11:40:10.608934Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"# used for training\ntraining_samples = []\nvalues_counts = []\nnum_classes = 0\n\n# used for training\ntraining_samples = []\nvalues_counts = []\nnum_classes = 0\n\n# H&M\nfiles = glob.glob(\"../H&M/images/*/*\")\nfile_paths = dict((os.path.splitext(os.path.split(f)[-1])[0], f) for f in files)\n\ndf = pd.read_csv('../H&M/articles.csv', \n                 usecols=['article_id', 'product_code'],\n                 dtype={'article_id': str, 'product_code': str})\n\ngroupped_products = {}\nfor index, row in df.iterrows():\n    v = groupped_products.get(row['product_code'], [])\n    f = file_paths.get(row['article_id'])\n    if f:\n        groupped_products[row['product_code']] = v + [f]\n\n\nfor key, value in groupped_products.items():\n    if len(value) >= CFG.min_samples:\n        paths = value[:CFG.samples_per_class]\n        \n        values_counts.append(len(paths))\n        training_samples.extend([\n            (p, num_classes) for p in paths\n        ])\n        num_classes += 1\n        \n        \n        \n\n# Product-10k\ndf = pd.read_csv('/kaggle/input/products-10k/products-10k/train.csv')\ndf_g = df.groupby('class', group_keys=True).apply(lambda x: x)\n\n\ntrain_df = pd.read_csv('/kaggle/input/products-10k/products-10k/train.csv')\ntrain_df['path'] = train_df.apply(lambda x: '/kaggle/input/products-10k/products-10k/train' + '/' + x['name'], axis=1)\n\n\n# # remove 9397815.jpg from the list!\n# test_df = pd.read_csv('/kaggle/input/test-data/test_kaggletest.csv')\n# test_df = test_df.drop(test_df[test_df.name == '9397815.jpg'].index) # smt wrong with this img\n# test_df['path'] = test_df.apply(lambda x: '/kaggle/input/products-10k/products-10k/test' + '/' + x['name'], axis=1)\n\n# df = pd.concat([\n#     test_df[['class','path']],\n#     train_df[['class', 'path']]\n# ])\n# df_g = df.groupby('class', group_keys=True).apply(lambda x: x)\n\nfor group in tqdm(set(train_df['class'])):\n    names = list(train_df.path[train_df['class'] == group])\n    if len(names) >= CFG.min_samples:\n        paths = [\n            name for name in names[:CFG.samples_per_class]\n        ]\n\n        values_counts.append(len(paths))\n        training_samples.extend([\n            (p, num_classes) for p in paths\n        ])\n\n        num_classes += 1\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:25.363302Z","iopub.execute_input":"2023-11-02T11:47:25.364224Z","iopub.status.idle":"2023-11-02T11:47:32.208555Z","shell.execute_reply.started":"2023-11-02T11:47:25.364185Z","shell.execute_reply":"2023-11-02T11:47:32.207472Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stderr","text":"/tmp/ipykernel_31/582126334.py:26: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\nPlease use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n  for group in tqdm(set(train_df['class'])):\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9691 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"793e979528414e088a1ac568d2815f32"}},"metadata":{}}]},{"cell_type":"code","source":"data_train = training_samples\nvalue_counts = np.array(values_counts)\nCFG.n_classes = num_classes","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:38.284963Z","iopub.execute_input":"2023-11-02T11:47:38.285855Z","iopub.status.idle":"2023-11-02T11:47:38.384898Z","shell.execute_reply.started":"2023-11-02T11:47:38.28582Z","shell.execute_reply":"2023-11-02T11:47:38.383813Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"len(data_train), CFG.n_classes","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:38.770256Z","iopub.execute_input":"2023-11-02T11:47:38.77116Z","iopub.status.idle":"2023-11-02T11:47:38.873078Z","shell.execute_reply.started":"2023-11-02T11:47:38.771124Z","shell.execute_reply":"2023-11-02T11:47:38.87206Z"},"trusted":true},"execution_count":19,"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"(139875, 9004)"},"metadata":{}}]},{"cell_type":"code","source":"class Head(nn.Module):\n    def __init__(self, hidden_size, k=3):\n        super(Head, self).__init__()\n        self.emb = nn.Linear(hidden_size, CFG.emb_size, bias=False)\n        self.dropout = Multisample_Dropout()\n        self.arc = ArcMarginProduct_subcenter(CFG.emb_size, CFG.n_classes, k)\n        \n    def forward(self, x):\n        embeddings = self.dropout(x, self.emb)\n        output = self.arc(embeddings)\n        return output, F.normalize(embeddings)\n    \nclass HeadV2(nn.Module):\n    def __init__(self, hidden_size, k=3):\n        super(HeadV2, self).__init__()\n        self.arc = ArcMarginProduct_subcenter(hidden_size, CFG.n_classes, k)\n        \n    def forward(self, x):\n        output = self.arc(x)\n        return output, F.normalize(x)\n    \nclass HeadV3(nn.Module):\n    def __init__(self, hidden_size, k=3):\n        super(HeadV3, self).__init__()        \n        self.emb = nn.Linear(hidden_size, CFG.emb_size, bias=False)\n        self.dropout = nn.Dropout1d(0.2)\n        self.arc = ArcMarginProduct_subcenter(CFG.emb_size, CFG.n_classes, k)\n        \n    def forward(self, x):\n        x = self.dropout(x)\n        x = self.emb(x)\n        output = self.arc(x)\n        return output, F.normalize(x)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:39.227879Z","iopub.execute_input":"2023-11-02T11:47:39.228235Z","iopub.status.idle":"2023-11-02T11:47:39.336689Z","shell.execute_reply.started":"2023-11-02T11:47:39.228204Z","shell.execute_reply":"2023-11-02T11:47:39.335769Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, vit_backbone, head_size, version='v1', k=3):\n        super(Model, self).__init__()\n        if version == 'v1':\n            self.head = Head(head_size, k)\n        elif version == 'v2':\n            self.head = HeadV2(head_size, k)\n        elif version == 'v3':\n            self.head = HeadV3(head_size, k)\n        else:\n            self.head = Head(head_size, k)\n        \n        self.encoder = vit_backbone.visual\n    def forward(self, x):\n        x = self.encoder(x)\n        return self.head(x)\n\n    def get_parameters(self):\n\n        parameter_settings = [] \n        parameter_settings.extend(\n            self.get_parameter_section(\n                [(n, p) for n, p in self.encoder.named_parameters()], \n                lr=CFG.vit_bb_lr, \n                wd=CFG.vit_bb_wd\n            )\n        ) \n\n        parameter_settings.extend(\n            self.get_parameter_section(\n                [(n, p) for n, p in self.head.named_parameters()], \n                lr=CFG.hd_lr, \n                wd=CFG.hd_wd\n            )\n        ) \n\n        return parameter_settings\n\n    def get_parameter_section(self, parameters, lr=None, wd=None): \n        parameter_settings = []\n\n\n        lr_is_dict = isinstance(lr, dict)\n        wd_is_dict = isinstance(wd, dict)\n\n        layer_no = None\n        for no, (n,p) in enumerate(parameters):\n            \n            for split in n.split('.'):\n                if split.isnumeric():\n                    layer_no = int(split)\n            \n            if not layer_no:\n                layer_no = 0\n            \n            if lr_is_dict:\n                for k,v in lr.items():\n                    if layer_no < int(k):\n                        temp_lr = v\n                        break\n            else:\n                temp_lr = lr\n\n            if wd_is_dict:\n                for k,v in wd.items():\n                    if layer_no < int(k):\n                        temp_wd = v\n                        break\n            else:\n                temp_wd = wd\n\n            weight_decay = 0.0 if 'bias' in n else temp_wd\n\n            parameter_setting = {\"params\" : p, \"lr\" : temp_lr, \"weight_decay\" : temp_wd}\n\n            parameter_settings.append(parameter_setting)\n\n            #print(f'no {no} | params {n} | lr {temp_lr} | weight_decay {weight_decay} | requires_grad {p.requires_grad}')\n\n        return parameter_settings","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:39.794001Z","iopub.execute_input":"2023-11-02T11:47:39.794825Z","iopub.status.idle":"2023-11-02T11:47:39.901778Z","shell.execute_reply.started":"2023-11-02T11:47:39.794795Z","shell.execute_reply":"2023-11-02T11:47:39.900919Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ArcFace_criterion(logits_m, target, margins):\n    arc = ArcFaceLossAdaptiveMargin(margins=margins, s=CFG.s, crit=CFG.crit)\n    loss_m = arc(logits_m, target, CFG.n_classes)\n    return loss_m","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:40.663479Z","iopub.execute_input":"2023-11-02T11:47:40.663844Z","iopub.status.idle":"2023-11-02T11:47:40.761476Z","shell.execute_reply.started":"2023-11-02T11:47:40.663816Z","shell.execute_reply":"2023-11-02T11:47:40.760561Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"def train(model, train_loader, optimizer, scaler, scheduler, epoch):\n    model.train()\n    loss_metrics = AverageMeter()\n    criterion = ArcFace_criterion\n\n    tmp = np.sqrt(1 / np.sqrt(value_counts))\n    margins = (tmp - tmp.min()) / (tmp.max() - tmp.min()) * CFG.m + CFG.m_min\n        \n    bar = tqdm(train_loader)\n    for step, data in enumerate(bar):\n        step += 1\n        images = data['images'].to(CFG.device, dtype=torch.float)\n        labels = data['labels'].to(CFG.device)\n        batch_size = labels.size(0)\n\n        with torch.cuda.amp.autocast(enabled=CFG.autocast):\n            outputs, features = model(images)\n\n        loss = criterion(outputs, labels, margins)\n        loss_metrics.update(loss.item(), batch_size)\n        loss = loss / CFG.acc_steps\n        scaler.scale(loss).backward()\n\n        if step % CFG.acc_steps == 0 or step == len(bar):\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            scheduler.step()\n            CFG.global_step += 1\n                        \n        lrs = get_lr_groups(optimizer.param_groups)\n\n        loss_avg = loss_metrics.avg\n\n        bar.set_postfix(loss=loss_avg, epoch=epoch, lrs=lrs, step=CFG.global_step)\n    \n@torch.no_grad()\ndef val(model, valid_loader):\n    model.eval() \n\n    all_embeddings = []\n    all_labels = [] \n\n    for data in tqdm(valid_loader):\n        images = data['images'].to(CFG.device, dtype=torch.float)\n        labels = data['labels'].to(CFG.device)\n\n        _, embeddings = model(images)\n\n        all_embeddings.append(embeddings.detach().cpu().numpy())\n        all_labels.append(labels.detach().cpu().numpy())\n\n\n    all_embeddings = np.concatenate(all_embeddings, axis=0)\n    all_labels = np.concatenate(all_labels, axis=0)\n\n    return all_embeddings, all_labels\n\ndef training(train_loader, \n             gallery_loader, \n             query_loader, \n             experiment_folder, \n             version='v1', \n             k=3, \n             reduce_lr_on_epoch=1,\n             use_rampup=True):\n    \n    os.makedirs(experiment_folder, exist_ok=True)\n    \n    backbone, _, _ = open_clip.create_model_and_transforms(CFG.model_name, CFG.model_data)\n\n    model = Model(backbone, CFG.hidden_layer, version, k).to(CFG.device)\n    \n    optimizer = torch.optim.AdamW(model.get_parameters())\n \n    scaler = torch.cuda.amp.GradScaler(enabled=CFG.autocast)\n\n    steps_per_epoch = math.ceil(len(train_loader) / CFG.acc_steps)\n\n    num_training_steps = math.ceil(CFG.n_epochs * steps_per_epoch)\n    \n    if use_rampup:\n        scheduler = get_cosine_schedule_with_warmup(optimizer,\n                                                    num_training_steps=num_training_steps,\n                                                    num_warmup_steps=CFG.n_warmup_steps)  \n    else:\n        scheduler = get_constant_schedule(optimizer)\n        \n    best_score = 0\n    best_updated_ = 0\n    CFG.global_step = 0                   \n    for epoch in range(math.ceil(CFG.n_epochs)):\n        print(f'starting epoch {epoch}')\n\n        # train of product-10k\n        train(model, train_loader, optimizer, scaler, scheduler, epoch)\n\n        # aicrowd test data\n        print('gallery embeddings')\n        embeddings_gallery, labels_gallery = val(model, gallery_loader)\n        print('query embeddings')\n        embeddings_query, labels_query = val(model, query_loader)\n\n        # idk why it is needed\n        gc.collect()\n        torch.cuda.empty_cache() \n\n        # calculate validation score\n        _, indices = get_similiarity_l2(embeddings_gallery, embeddings_query, 1000)\n\n\n        indices = indices.tolist()\n        labels_gallery = labels_gallery.tolist()\n        labels_query = labels_query.tolist()\n\n        preds = convert_indices_to_labels(indices, labels_gallery)\n        score = map_per_set(labels_query, preds)\n        print('validation score', score)\n\n        # save model\n        torch.save({\n                'model_state_dict': model.state_dict(),\n                }, f'{experiment_folder}/model_epoch_{epoch+1}_mAP3_{score:.2f}.pt')\n\n        # early stopping\n        if score > best_score:\n            best_updated_ = 0\n            best_score = score\n\n        best_updated_ += 1\n\n        if best_updated_ >= 3:\n            print('no improvement done training....')\n            break\n            \n        if (epoch + 1) % reduce_lr_on_epoch == 0:\n            scheduler.base_lrs = [g['lr'] * CFG.reduce_lr for g in optimizer.param_groups]\n            \n        # to speed up the training\n        if epoch > 3:\n            break","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:42.789389Z","iopub.execute_input":"2023-11-02T11:47:42.789755Z","iopub.status.idle":"2023-11-02T11:47:42.905666Z","shell.execute_reply.started":"2023-11-02T11:47:42.789724Z","shell.execute_reply":"2023-11-02T11:47:42.90462Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"# data loader\n\ndef read_img(img_path, is_gray=False):\n    mode = cv2.IMREAD_COLOR if not is_gray else cv2.IMREAD_GRAYSCALE\n    img = cv2.imread(img_path, mode)\n    if not is_gray:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\n\ndef get_final_transform():  \n    final_transform = T.Compose([\n            T.Resize(\n                size=(CFG.image_size, CFG.image_size), \n                interpolation=T.InterpolationMode.BICUBIC,\n                antialias=True),\n            T.ToTensor(), \n            T.Normalize(\n                mean=(0.48145466, 0.4578275, 0.40821073), \n                std=(0.26862954, 0.26130258, 0.27577711)\n            )\n        ])\n    return final_transform\n\nclass ProductDataset(Dataset):\n    def __init__(self, \n                 data, \n                 transform=None, \n                 final_transform=None):\n        self.data = data\n        self.transform = transform\n        self.final_transform = final_transform\n            \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n       \n        img = read_img(self.data[idx][0])            \n        \n        if self.transform is not None:\n            if isinstance(self.transform, A.Compose):\n                img = self.transform(image=img)['image']\n            else:\n                img = self.transform(img)\n        \n        if self.final_transform is not None:\n            if isinstance(img, np.ndarray):\n                img =  Image.fromarray(img)\n            img = self.final_transform(img)\n            \n        product_id = self.data[idx][1]\n        return {\"images\": img, \"labels\": product_id}\n    \ndef get_product_10k_dataloader(data_train, data_aug='image_net'):\n    \n    transform = None\n    if data_aug == 'image_net':\n        transform = T.Compose([\n            T.ToPILImage(),\n            T.AutoAugment(T.AutoAugmentPolicy.IMAGENET)\n        ])\n        \n    elif data_aug == 'aug_mix':\n        transform = T.Compose([\n            T.ToPILImage(),\n            T.AugMix()\n        ])\n    elif data_aug == 'happy_whale':\n        aug8p3 = A.OneOf([\n            A.Sharpen(p=0.3),\n            A.ToGray(p=0.3),\n            A.CLAHE(p=0.3),\n        ], p=0.5)\n\n        transform = A.Compose([\n            A.ShiftScaleRotate(rotate_limit=15, scale_limit=0.1, border_mode=cv2.BORDER_REFLECT, p=0.5),\n            A.Resize(CFG.image_size, CFG.image_size),\n            aug8p3,\n            A.HorizontalFlip(p=0.5),\n            A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1)\n        ])\n    \n    elif data_aug == 'cut_out':        \n        transform = A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.ImageCompression(quality_lower=99, quality_upper=100),\n            A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n            A.Resize(CFG.image_size, CFG.image_size),\n            A.Cutout(max_h_size=int(CFG.image_size * 0.4), \n                     max_w_size=int(CFG.image_size * 0.4), \n                     num_holes=1, p=0.5),\n        ])\n    elif data_aug == 'clip':\n        transform = T.Compose([\n            T.ToPILImage(),\n            T.RandomResizedCrop(\n                size=(224, 224), \n                scale=(0.9, 1.0), \n                ratio=(0.75, 1.3333), \n                interpolation=T.InterpolationMode.BICUBIC,\n                antialias=True\n            )\n        ])\n    elif data_aug == 'clip+image_net':\n        transform = T.Compose([\n            T.ToPILImage(),\n            T.AutoAugment(T.AutoAugmentPolicy.IMAGENET),\n            T.RandomResizedCrop(\n                size=(224, 224), \n                scale=(0.9, 1.0), \n                ratio=(0.75, 1.3333), \n                interpolation=T.InterpolationMode.BICUBIC,\n                antialias=True\n            )\n        ])\n    \n    final_transform = get_final_transform()\n    train_dataset = ProductDataset(data_train, \n                                   transform, \n                                   final_transform)\n    train_loader = DataLoader(train_dataset, \n                              batch_size = CFG.train_batch_size, \n                              num_workers=CFG.workers, \n                              shuffle=True, \n                              drop_last=True)\n    print(f'Training Data -> Dataset Length ({len(train_dataset)})')\n    return train_loader\n\ndef aicrowd_data_loader(csv_path, img_dir='/kaggle/input/products-10k/products-10k/development_test_data'):\n    df_g = pd.read_csv(csv_path)\n    df_g_ = df_g[['img_path', 'product_id']]\n    df_g_['img_path'] = df_g_.apply(lambda x: img_dir + '/' + x['img_path'], axis=1)\n    data_ = np.array(df_g_).tolist()\n    \n    final_transform = get_final_transform()\n    dataset = ProductDataset(data_, None, final_transform)\n    data_loader = DataLoader(dataset, \n                             batch_size = CFG.valid_batch_size, \n                             num_workers=CFG.workers, \n                             shuffle=False, \n                             drop_last=False)\n    \n    print(f'{csv_path} -> Dataset Length ({len(dataset)})')\n    return data_loader","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:43.23616Z","iopub.execute_input":"2023-11-02T11:47:43.236562Z","iopub.status.idle":"2023-11-02T11:47:43.351428Z","shell.execute_reply.started":"2023-11-02T11:47:43.23653Z","shell.execute_reply":"2023-11-02T11:47:43.35043Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"# aicrowd datasets\ngallery_loader = aicrowd_data_loader('/kaggle/input/products-10k/products-10k/development_test_data/gallery.csv')\nquery_loader = aicrowd_data_loader('/kaggle/input/products-10k/products-10k/development_test_data/queries.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:43.862869Z","iopub.execute_input":"2023-11-02T11:47:43.863229Z","iopub.status.idle":"2023-11-02T11:47:44.027186Z","shell.execute_reply.started":"2023-11-02T11:47:43.863199Z","shell.execute_reply":"2023-11-02T11:47:44.026321Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"/kaggle/input/products-10k/products-10k/development_test_data/gallery.csv -> Dataset Length (1067)\n/kaggle/input/products-10k/products-10k/development_test_data/queries.csv -> Dataset Length (1935)\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_31/4194327427.py:132: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame.\nTry using .loc[row_indexer,col_indexer] = value instead\n\nSee the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n  df_g_['img_path'] = df_g_.apply(lambda x: img_dir + '/' + x['img_path'], axis=1)\n/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 12 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n  warnings.warn(_create_warning_msg(\n/tmp/ipykernel_31/4194327427.py:132: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame.\nTry using .loc[row_indexer,col_indexer] = value instead\n\nSee the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n  df_g_['img_path'] = df_g_.apply(lambda x: img_dir + '/' + x['img_path'], axis=1)\n/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 12 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n  warnings.warn(_create_warning_msg(\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k = 3  \nversion = 'v2'\ndata_aug = 'image_net'\nCFG.reduce_lr = 0.1\ntrain_loader = get_product_10k_dataloader(data_train, data_aug)\nexperiment_folder = f'my_experiments/{CFG.model_name}-{CFG.model_data}-{str(data_aug)}-{str(version)}-p10k-h&m-Arcface(k={str(k)})-All-Epoch({str(CFG.n_epochs)})-Reduce_LR_0.1'\ntraining(train_loader, \n         gallery_loader, \n         query_loader, \n         experiment_folder, \n         version=version,\n         k=k)\n# idk why it is needed\ngc.collect()\ntorch.cuda.empty_cache() ","metadata":{"execution":{"iopub.status.busy":"2023-11-02T11:47:45.324358Z","iopub.execute_input":"2023-11-02T11:47:45.325458Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Training Data -> Dataset Length (139875)\n","output_type":"stream"},{"name":"stderr","text":"100%|████████████████████████████████████████| 605M/605M [00:02<00:00, 298MiB/s]\n","output_type":"stream"},{"name":"stdout","text":"starting epoch 0\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_31/69089968.py:9: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\nPlease use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n  bar = tqdm(train_loader)\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/17484 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ccca1651d9654d6ea47e8698f0f5b309"}},"metadata":{}},{"name":"stdout","text":"gallery embeddings\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_31/69089968.py:44: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\nPlease use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n  for data in tqdm(valid_loader):\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/34 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a86436f3665b4bda8079270e3e4ff3fe"}},"metadata":{}},{"name":"stdout","text":"query embeddings\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/61 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"64a3099e6cb84fa4a79aad27007d0e77"}},"metadata":{}},{"name":"stdout","text":"Processing indices...\nFinished processing indices, took 0.26970553398132324s\nvalidation score 0.31763996554694235\nstarting epoch 1\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/17484 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"04d8220db2964aae99cbff39c30c6edd"}},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}