{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71885,"databundleVersionId":8143495,"sourceType":"competition"},{"sourceId":8367334,"sourceType":"datasetVersion","datasetId":4973903},{"sourceId":8367624,"sourceType":"datasetVersion","datasetId":4974117},{"sourceId":8368028,"sourceType":"datasetVersion","datasetId":4974104},{"sourceId":8368095,"sourceType":"datasetVersion","datasetId":4974284},{"sourceId":45794,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":38394}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nIn this note book, I try implementing part of the solution gaining the [2nd position at IMC2023](https://www.kaggle.com/competitions/image-matching-challenge-2023/discussion/416873). The authors had used this [solution](https://github.com/ternaus/check_orientation) to correct rotated images before using it in the rest of the pipeline. \n\nThis notebook allows user to explore the dataset of choice. Then run the rotation correction and visualize the outputs.","metadata":{}},{"cell_type":"markdown","source":"# Import Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/mediapy/mediapy mediapy\n!pip install --no-index /kaggle/input/check-orientation/check-orientation/iglovikov_helper_functions-0.0.53-py2.py3-none-any.whl --no-deps\n\nimport os\nimport numpy as np\nimport mediapy as media\nimport cv2\nfrom glob import glob\nfrom pathlib import Path\nfrom time import time\nfrom pprint import pprint\nimport os\nimport shutil\nimport torch\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom collections import namedtuple\nfrom typing import Optional\nfrom iglovikov_helper_functions.dl.pytorch.utils import rename_layers\nfrom timm import create_model as timm_create_model\nfrom torch import nn\n","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:08:04.105854Z","iopub.execute_input":"2024-05-09T17:08:04.106206Z","iopub.status.idle":"2024-05-09T17:08:25.929069Z","shell.execute_reply.started":"2024-05-09T17:08:04.106176Z","shell.execute_reply":"2024-05-09T17:08:25.928306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True;","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:08:25.930786Z","iopub.execute_input":"2024-05-09T17:08:25.931062Z","iopub.status.idle":"2024-05-09T17:08:25.937051Z","shell.execute_reply.started":"2024-05-09T17:08:25.931038Z","shell.execute_reply":"2024-05-09T17:08:25.936246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copy Work Directory\nCreate a new working directory to work with.\n","metadata":{}},{"cell_type":"code","source":"# Source and destination paths\nsource_path = \"/kaggle/input/image-matching-challenge-2024\"\ndestination_path = \"/kaggle/working/image-matching-challenge-2024\"\n\nif not os.path.exists(destination_path):\n    shutil.copytree(source_path, destination_path)\n    print(\"Directory copied successfully!\")\nelse:\n    print(\"Destination directory already exists. Skipping the copying process.\")","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:08:25.938045Z","iopub.execute_input":"2024-05-09T17:08:25.938352Z","iopub.status.idle":"2024-05-09T17:09:29.217981Z","shell.execute_reply.started":"2024-05-09T17:08:25.938329Z","shell.execute_reply":"2024-05-09T17:09:29.217057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Uses this section for testing \n# Use the dioscuri together with church as church dont have any disoriented photos\n\nif DEBUG:\n    # Source and destination directories\n    source_dir = \"/kaggle/working/image-matching-challenge-2024/train/dioscuri/images\"\n    destination_dir = \"/kaggle/working/image-matching-challenge-2024/test/dioscuri/images\"\n\n    # Remove the existing destination directory\n    #shutil.rmtree(destination_dir)\n\n    # Copy the entire directory tree from source to destination\n    shutil.copytree(source_dir, destination_dir)\n\n    print(\"Directory replaced successfully.\")","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:09:29.219270Z","iopub.execute_input":"2024-05-09T17:09:29.219574Z","iopub.status.idle":"2024-05-09T17:09:29.295654Z","shell.execute_reply.started":"2024-05-09T17:09:29.219549Z","shell.execute_reply":"2024-05-09T17:09:29.294619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Check Orientation model\nInput an image \n\nOutput a prediction in 4 rotation class [0deg 90deg 180deg 270deg]\n\nusing the MODELS at /kaggle/input/swsl_resnext50_32x4d/pytorch/v1/1","metadata":{}},{"cell_type":"code","source":"def create_model(model_name: str, activation: Optional[str] = \"softmax\") -> nn.Module:\n    # Define the 'model' namedtuple\n    model = namedtuple(\"model\", [\"url\", \"model\"])\n\n    # Define the models dictionary\n    models = {\n        \"swsl_resnext50_32x4d\": model(\n            model=timm_create_model(\"swsl_resnext50_32x4d\", pretrained=False, num_classes=4),\n            url=\"/kaggle/input/swsl_resnext50_32x4d/pytorch/v1/1/2020-11-16_resnext50_32x4d.pth\",\n        ),\n    }\n\n    # Load the model from models dictionary\n    model = models[model_name].model\n    \n    # Load the model state dictionary from local path\n    state_dict = torch.load(models[model_name].url, map_location=\"cpu\")[\"state_dict\"]\n    state_dict = rename_layers(state_dict, {\"model.\": \"\"})\n    # Load the state dictionary into the model\n    model.load_state_dict(state_dict)\n\n    if activation == \"softmax\":\n        return nn.Sequential(model, nn.Softmax(dim=1))\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:25.182171Z","iopub.execute_input":"2024-05-09T17:24:25.182901Z","iopub.status.idle":"2024-05-09T17:24:25.189979Z","shell.execute_reply.started":"2024-05-09T17:24:25.182871Z","shell.execute_reply":"2024-05-09T17:24:25.189039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(\"swsl_resnext50_32x4d\")\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:25.401975Z","iopub.execute_input":"2024-05-09T17:24:25.402278Z","iopub.status.idle":"2024-05-09T17:24:25.934973Z","shell.execute_reply.started":"2024-05-09T17:24:25.402240Z","shell.execute_reply":"2024-05-09T17:24:25.934047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the images\n\nIn this section, I iterate through each of the directories(scene) of images and display them. I also counted the number of images that needed rotation correction. Looks like only \"heritage/dioscuri\" and \"heritage/cyprus\" dataset have a lot of rotated images. Please change the `dirIdx` in the following code cell to view the data in that directory.","metadata":{}},{"cell_type":"code","source":"# Path list for all the scene of the test set\nimagesDirList_test = glob('/kaggle/working/image-matching-challenge-2024/test/*/images')\npprint(imagesDirList_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:30.678177Z","iopub.execute_input":"2024-05-09T17:24:30.678880Z","iopub.status.idle":"2024-05-09T17:24:30.683962Z","shell.execute_reply.started":"2024-05-09T17:24:30.678851Z","shell.execute_reply":"2024-05-09T17:24:30.683041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    for dirIdx in range(len(imagesDirList_test)):\n        src = imagesDirList_test[dirIdx]\n        images = [cv2.cvtColor(cv2.imread(im), cv2.COLOR_BGR2RGB) for im in glob(f'{src}/*')]\n        print(f'Showing {len(images)} image from {src}')\n        media.show_images(images, height=300, columns=5)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:11:11.736860Z","iopub.execute_input":"2024-05-09T17:11:11.737519Z","iopub.status.idle":"2024-05-09T17:11:21.039750Z","shell.execute_reply.started":"2024-05-09T17:11:11.737479Z","shell.execute_reply":"2024-05-09T17:11:21.036790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a Torch Dataset\nIn order to pass that data as input to the NN, we needed to convert to tensor and apply some transforms. I decided create a torch dataset that will iterate through the images in a given folder.","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torchvision.io import read_image\n\nclass IMC2024TestSet(Dataset):\n    def __init__(self, imgDir, transform=None):\n        #this is the scene folder\n        self.imgDir = imgDir\n        self.transform = transform\n        self.imgPathList = [imgPath for imgPath in glob(f'{imgDir}/*')]\n\n    def __len__(self):\n        return len(self.imgPathList)\n\n    def __getitem__(self, idx):\n        imgPath = self.imgPathList[idx]\n        image = read_image(imgPath)\n        if self.transform:\n            image = self.transform(image)\n        return imgPath, image","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:36.627680Z","iopub.execute_input":"2024-05-09T17:24:36.628325Z","iopub.status.idle":"2024-05-09T17:24:36.635338Z","shell.execute_reply.started":"2024-05-09T17:24:36.628283Z","shell.execute_reply":"2024-05-09T17:24:36.634373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision import transforms as T\n\ntransform = T.Compose([\n    T.Resize((224, 224)),\n    T.ConvertImageDtype(torch.float),\n    T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:37.337776Z","iopub.execute_input":"2024-05-09T17:24:37.338139Z","iopub.status.idle":"2024-05-09T17:24:37.343687Z","shell.execute_reply.started":"2024-05-09T17:24:37.338108Z","shell.execute_reply":"2024-05-09T17:24:37.342562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Rotation Correction\nIn this section, we correct the rotation of the images.","metadata":{}},{"cell_type":"code","source":"for sceneDir in imagesDirList_test:\n    \n    scene = IMC2024TestSet(sceneDir, transform=transform)\n    numberOfImagesRotated = 0\n    \n    for idx, out in enumerate(scene):\n        imgPath, image = out\n#         print(f'{idx}')\n#         print(f'{imgPath}')\n\n#         if DEBUG:\n#             # Load and display the image using Matplotlib\n#             img = mpimg.imread(imgPath)\n#             plt.imshow(img)\n#             plt.axis('off')  # Hide axes\n#             plt.show()\n\n        with torch.no_grad():\n            prediction = model(image[None, ...]).numpy()[0]\n            \n        predictedRotationDeg = 360 - (prediction.argmax() * 90)\n        \n        if predictedRotationDeg == 360:\n            print(f'{idx}. {imgPath} :`s orientation is correct.')\n        else:\n            # Read the image\n            print(f'{idx}. {imgPath} :`s ROTATED.')\n            numberOfImagesRotated += 1\n            \n            # Open the PNG image\n            image = Image.open(imgPath)\n            # Rotate the image by \"predictedRotationDeg\" (90 180 270) degrees clockwise (use expand=True to avoid cropping)\n            rotated_image = image.rotate(predictedRotationDeg, expand=True)\n            if DEBUG:\n                # Display both the original and rotated images using Matplotlib\n                fig, axes = plt.subplots(1, 2, figsize=(10, 5))\n\n                # Display original image\n                axes[0].imshow(image)\n                axes[0].set_title(\"Original Image\")\n                axes[0].axis('off')\n\n                # Display rotated image\n                axes[1].imshow(rotated_image)\n                axes[1].set_title(\"Rotated Image\")\n                axes[1].axis('off')\n\n                plt.show()\n            # Save the rotated image to the same path to overwrite the original image\n            rotated_image.save(imgPath)\n\n    print(f\"Total {numberOfImagesRotated} in {sceneDir} corrected.\")","metadata":{"execution":{"iopub.status.busy":"2024-05-09T17:24:38.718763Z","iopub.execute_input":"2024-05-09T17:24:38.719730Z","iopub.status.idle":"2024-05-09T17:25:27.289440Z","shell.execute_reply.started":"2024-05-09T17:24:38.719694Z","shell.execute_reply":"2024-05-09T17:25:27.288376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PROBLEMS: the models must be downloaded from github (the .tar file) so i will download it to kaggle manueallly instead and define anew fn to use it.\n","metadata":{}}]}