{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# XYMasking aug in Albumentations","metadata":{}},{"cell_type":"markdown","source":"In [Albumentations](https://albumentations.ai/) library we have many image related augmentations, but, apparantely there was none that could do the same as [TimeMasking](https://pytorch.org/audio/main/generated/torchaudio.transforms.TimeMasking.html) [FrequencyMasking](https://pytorch.org/audio/main/generated/torchaudio.transforms.FrequencyMasking.html) from torchaudio.\n\nTechnically it is possible to create such transforms for this competition using [LambdaTransform](https://albumentations.ai/docs/api_reference/augmentations/transforms/#albumentations.augmentations.transforms.Lambda) where you define python function that should be applied to images, so, for all practical purposes you do not need special transform.\n\nBut:\n1. Native tranforms could be serialized /deserialized to json, yaml and python dictionaries, which is not the case for Lambda.\n2. Audio and eeg data becomes more and more popular => it makes sense to create something that works out of the box.\n3. The architecture of native masking transforms ([CoarseDropout](https://albumentations.ai/docs/api_reference/augmentations/dropout/coarse_dropout/), [MaskDropout](https://albumentations.ai/docs/api_reference/augmentations/dropout/mask_dropout/), [GridDropout](https://albumentations.ai/docs/api_reference/augmentations/dropout/grid_dropout/)) allows the transform to be applied in a more advanced settings that I will describe in this kernel.","metadata":{}},{"cell_type":"markdown","source":"## XYMasking","metadata":{}},{"cell_type":"markdown","source":"We added [XYMasking](https://albumentations.ai/docs/api_reference/augmentations/dropout/xy_masking/) in version 1.4.0 (released on 17 Feb 2024), it may take some time for Kaggle docker to update.\n","metadata":{}},{"cell_type":"code","source":"%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:29.994046Z","iopub.execute_input":"2024-02-18T01:08:29.994455Z","iopub.status.idle":"2024-02-18T01:08:30.001793Z","shell.execute_reply.started":"2024-02-18T01:08:29.994426Z","shell.execute_reply":"2024-02-18T01:08:30.000302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import *\nimport torchaudio\nimport torch\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:30.004724Z","iopub.execute_input":"2024-02-18T01:08:30.005732Z","iopub.status.idle":"2024-02-18T01:08:34.184318Z","shell.execute_reply.started":"2024-02-18T01:08:30.005687Z","shell.execute_reply":"2024-02-18T01:08:34.183084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U albumentations","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:34.185609Z","iopub.execute_input":"2024-02-18T01:08:34.186093Z","iopub.status.idle":"2024-02-18T01:08:50.260224Z","shell.execute_reply.started":"2024-02-18T01:08:34.186064Z","shell.execute_reply":"2024-02-18T01:08:50.258631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:50.263409Z","iopub.execute_input":"2024-02-18T01:08:50.263827Z","iopub.status.idle":"2024-02-18T01:08:51.467011Z","shell.execute_reply.started":"2024-02-18T01:08:50.263782Z","shell.execute_reply":"2024-02-18T01:08:51.466037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def spectrogram_from_eeg4(parquet_path: Path) -> np.ndarray:\n    NAMES = [\"LL\", \"LP\", \"RP\", \"RR\"]\n\n    FEATS = [\n        [\"Fp1\", \"F7\", \"T3\", \"T5\", \"O1\"],\n        [\"Fp1\", \"F3\", \"C3\", \"P3\", \"O1\"],\n        [\"Fp2\", \"F8\", \"T4\", \"T6\", \"O2\"],\n        [\"Fp2\", \"F4\", \"C4\", \"P4\", \"O2\"],\n    ]\n    \n    FOUR = 4\n    \n    # Load middle 50 seconds of EEG series\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg) - 10_000) // 2\n    eeg = eeg.iloc[middle : middle + 10_000]\n\n    # Variable to hold spectrogram\n    img = np.zeros((128, 256, 4), dtype=\"float32\")\n\n    for k in range(4):\n        cols = FEATS[k]\n\n        for kk in range(4):\n            # Compute pair differences\n            x = eeg[cols[kk]].to_numpy() - eeg[cols[kk + 1]].to_numpy()\n\n            # Fill NaNs\n            m = np.nanmean(x)\n            x = np.where(np.isnan(x), m, x)  # Vectorized operation for replacing NaNs\n\n            # Convert to tensor and add a batch dimension\n            x_tensor = torch.tensor(x, dtype=torch.float32).unsqueeze(0)\n\n            # Create MelSpectrogram object\n            mel_spectrogram = torchaudio.transforms.MelSpectrogram(\n                sample_rate=200,\n                n_fft=1024,\n                win_length=128,\n                hop_length=len(x) // 256,\n                n_mels=128,\n                f_min=0,\n                f_max=20,\n                power=2.0,\n            )\n\n            # Compute spectrogram\n            mel_spec_tensor = mel_spectrogram(x_tensor)\n\n            # Convert power spectrogram to dB scale\n            mel_spec_db_tensor = torchaudio.transforms.AmplitudeToDB(stype=\"power\")(mel_spec_tensor)\n\n            # Ensure the spectrogram is the expected shape\n            width = min(mel_spec_db_tensor.shape[2], 256)\n            mel_spec_db_tensor = mel_spec_db_tensor[:, :, :width].squeeze(0)  # Remove batch dimension\n\n            # Standardize to -1 to 1\n            mel_spec_db_np = (mel_spec_db_tensor.numpy() + 40) / 40\n            img[:, :width, k] += mel_spec_db_np\n\n        # Average the 4 montage differences\n        img[:, :width, k] /= 4.0\n\n    return img[::-1]","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.468621Z","iopub.execute_input":"2024-02-18T01:08:51.469069Z","iopub.status.idle":"2024-02-18T01:08:51.484100Z","shell.execute_reply.started":"2024-02-18T01:08:51.469042Z","shell.execute_reply":"2024-02-18T01:08:51.482880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_file_path = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/2208063991.parquet\"","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.485400Z","iopub.execute_input":"2024-02-18T01:08:51.486057Z","iopub.status.idle":"2024-02-18T01:08:51.499019Z","shell.execute_reply.started":"2024-02-18T01:08:51.485986Z","shell.execute_reply":"2024-02-18T01:08:51.497807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = spectrogram_from_eeg4(eeg_file_path)","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.500461Z","iopub.execute_input":"2024-02-18T01:08:51.501323Z","iopub.status.idle":"2024-02-18T01:08:51.963663Z","shell.execute_reply.started":"2024-02-18T01:08:51.501281Z","shell.execute_reply":"2024-02-18T01:08:51.962682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Spectrogram shape = \", img.shape)\nprint(img.min(), img.max())","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.965002Z","iopub.execute_input":"2024-02-18T01:08:51.965287Z","iopub.status.idle":"2024-02-18T01:08:51.971302Z","shell.execute_reply.started":"2024-02-18T01:08:51.965262Z","shell.execute_reply":"2024-02-18T01:08:51.970276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = np.ascontiguousarray(img)","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.975176Z","iopub.execute_input":"2024-02-18T01:08:51.975524Z","iopub.status.idle":"2024-02-18T01:08:51.982559Z","shell.execute_reply.started":"2024-02-18T01:08:51.975469Z","shell.execute_reply":"2024-02-18T01:08:51.981336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imshow(img[:, :, 0])","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:51.984000Z","iopub.execute_input":"2024-02-18T01:08:51.984386Z","iopub.status.idle":"2024-02-18T01:08:52.332746Z","shell.execute_reply.started":"2024-02-18T01:08:51.984351Z","shell.execute_reply":"2024-02-18T01:08:52.331610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One vertical stripe (time masking) with fixed width, filled with 0 ","metadata":{}},{"cell_type":"code","source":"params1 = {\n    \"num_masks_x\": 1,    \n    \"mask_x_length\": 20,\n    \"fill_value\": 0,    \n\n}\ntransform1 = A.Compose([A.XYMasking(**params1, p=1)])\nimshow(transform1(image=img[:, :, 0])[\"image\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:52.334232Z","iopub.execute_input":"2024-02-18T01:08:52.334664Z","iopub.status.idle":"2024-02-18T01:08:52.617912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One vertical stripe (time masking) with randomly sampled width, filled with 0","metadata":{}},{"cell_type":"code","source":"params2 = {\n    \"num_masks_x\": 1,    \n    \"mask_x_length\": (0, 20), # This line changed from fixed  to a range\n    \"fill_value\": 0,\n}\ntransform2 = A.Compose([A.XYMasking(**params2, p=1)])\nimshow(transform2(image=img[:, :, 0])[\"image\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:52.619516Z","iopub.execute_input":"2024-02-18T01:08:52.620628Z","iopub.status.idle":"2024-02-18T01:08:52.903020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Analogous transform in torchaudio","metadata":{}},{"cell_type":"code","source":"spectrogram = torchaudio.transforms.Spectrogram()\nmasking = torchaudio.transforms.TimeMasking(time_mask_param=20)\nmasked = masking(torch.from_numpy(img[:, :, 0]))\nimshow(masked.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:52.904730Z","iopub.execute_input":"2024-02-18T01:08:52.905047Z","iopub.status.idle":"2024-02-18T01:08:53.203093Z","shell.execute_reply.started":"2024-02-18T01:08:52.905021Z","shell.execute_reply":"2024-02-18T01:08:53.201932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One horizontal stripe (frequency masking) with randomly sampled width, filled with 0","metadata":{}},{"cell_type":"code","source":"params3 = {    \n    \"num_masks_y\": 1,    \n    \"mask_y_length\": (0, 20),\n    \"fill_value\": 0,    \n\n}\ntransform3 = A.Compose([A.XYMasking(**params3, p=1)])\nimshow(transform3(image=img[:, :, 0])[\"image\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:53.204816Z","iopub.execute_input":"2024-02-18T01:08:53.205546Z","iopub.status.idle":"2024-02-18T01:08:53.495229Z","shell.execute_reply.started":"2024-02-18T01:08:53.205507Z","shell.execute_reply":"2024-02-18T01:08:53.494255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Analogous transform in torchaudio","metadata":{}},{"cell_type":"code","source":"spectrogram = torchaudio.transforms.Spectrogram()\nmasking = torchaudio.transforms.FrequencyMasking(freq_mask_param=20)\nmasked = masking(torch.from_numpy(img[:, :, 0]))\nimshow(masked)","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:53.496695Z","iopub.execute_input":"2024-02-18T01:08:53.497029Z","iopub.status.idle":"2024-02-18T01:08:53.790414Z","shell.execute_reply.started":"2024-02-18T01:08:53.497001Z","shell.execute_reply":"2024-02-18T01:08:53.789305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Several vertical and horizontal stripes","metadata":{}},{"cell_type":"code","source":"params4 = {    \n    \"num_masks_x\": (2, 4),\n    \"num_masks_y\": 5,    \n    \"mask_y_length\": 8,\n    \"mask_x_length\": (10, 20),\n    \"fill_value\": 0,  \n\n}\ntransform4 = A.Compose([A.XYMasking(**params4, p=1)])\nimshow(transform4(image=img[:, :, 0])[\"image\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:53.792050Z","iopub.execute_input":"2024-02-18T01:08:53.793180Z","iopub.status.idle":"2024-02-18T01:08:54.069398Z","shell.execute_reply.started":"2024-02-18T01:08:53.793137Z","shell.execute_reply":"2024-02-18T01:08:54.068413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Application to the image with the number of channels larger than 3, and different fill values for different channels ","metadata":{}},{"cell_type":"markdown","source":"Transform can work with any number of channels supporing image shapes of \n\n* Grayscale: (height, width)\n* RGB: (height, width, 3)\n* Multichannel: (heigh, width, num_channels)\n\nFor value that is used to fill masking regions you can use:\n* scalar that will be applied to every channel\n* list of numbers equal to the number of channels, so that every channel will have it's own filling value","metadata":{}},{"cell_type":"code","source":"params5 = {    \n    \"num_masks_x\": (2, 4),\n    \"num_masks_y\": 5,    \n    \"mask_y_length\": 8,\n    \"mask_x_length\": (20, 30),\n    \"fill_value\": (0, 1, 2, 3),  \n\n}\ntransform5 = A.Compose([A.XYMasking(**params5, p=1)])\ntransformed = transform5(image=img)[\"image\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:54.070927Z","iopub.execute_input":"2024-02-18T01:08:54.071257Z","iopub.status.idle":"2024-02-18T01:08:54.078085Z","shell.execute_reply.started":"2024-02-18T01:08:54.071229Z","shell.execute_reply":"2024-02-18T01:08:54.076984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a figure and 4 subplots (1 row, 4 columns)\nfig, axs = plt.subplots(2, 2) # Adjust figsize to fit your needs\n\nvmin=0\nvmax=3\n\naxs[0, 0].imshow(transformed[:, :, 0], vmin=vmin, vmax=vmax)\naxs[0, 0].set_title('Channel 0')\naxs[0, 0].axis('off')  # Hide axes for cleaner visualization\n\naxs[0, 1].imshow(transformed[:, :, 1], vmin=vmin, vmax=vmax)\naxs[0, 1].set_title('Channel 1')\naxs[0, 1].axis('off')\n\naxs[1, 0].imshow(transformed[:, :, 2], vmin=vmin, vmax=vmax)\naxs[1, 0].set_title('Channel 2')\naxs[1, 0].axis('off')\n\naxs[1, 1].imshow(transformed[:, :, 3], vmin=vmin, vmax=vmax)\naxs[1, 1].set_title('Channel 3')\naxs[1, 1].axis('off')\n\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2024-02-18T01:08:54.079603Z","iopub.execute_input":"2024-02-18T01:08:54.079983Z","iopub.status.idle":"2024-02-18T01:08:54.508289Z","shell.execute_reply.started":"2024-02-18T01:08:54.079956Z","shell.execute_reply":"2024-02-18T01:08:54.507126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusion\n\nIf you have any questions, proposals, feature requests \n\n1. Feel free to write them to [issues in repository](https://github.com/albumentations-team/albumentations/issues) \nor\n2. Join [Discord server](https://discord.gg/AmMnDBdzYs)\nor\n3. Write me directly on [LinkedIn](https://www.linkedin.com/in/iglovikov/)\n\n**And the most important - I hope that this transform will be helpful in this competition!**","metadata":{}}]}