{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":11186015,"sourceType":"datasetVersion","datasetId":6982771},{"sourceId":11186167,"sourceType":"datasetVersion","datasetId":6982883},{"sourceId":11186248,"sourceType":"datasetVersion","datasetId":6982954},{"sourceId":11186369,"sourceType":"datasetVersion","datasetId":6983041},{"sourceId":11186434,"sourceType":"datasetVersion","datasetId":6983086},{"sourceId":11186443,"sourceType":"datasetVersion","datasetId":6983093},{"sourceId":11186475,"sourceType":"datasetVersion","datasetId":6983116},{"sourceId":282676,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":242214,"modelId":263840}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"target_tranform = {\"label_classification\": \"top_k(1, None)\"}\nfrom pathlib import Path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:09.530689Z","iopub.execute_input":"2025-03-27T13:45:09.531057Z","iopub.status.idle":"2025-03-27T13:45:09.535341Z","shell.execute_reply.started":"2025-03-27T13:45:09.531024Z","shell.execute_reply":"2025-03-27T13:45:09.534101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\n\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\n\ntry:\n    import os\n    import pandas as pd\n    # <｜fim▁begin｜>\n    import os\n    import pandas as pd\n    # Define the base directory\n    base_dir = '/kaggle/input/cassava-leaf-disease-classification'\n    # List all image files in the test_images directory\n    test_image_dir = os.path.join(base_dir, 'test_images')\n    image_files = [f for f in os.listdir(test_image_dir) if f.endswith('.jpg')]\n    # Create a DataFrame with the image IDs (including the .jpg extension) and full paths\n    test_img_input_map = pd.DataFrame({\n        'id': image_files,\n        'img_input1': [os.path.join(test_image_dir, f) for f in image_files]\n    })\n    test_img_input_map.to_csv(\"./test_img_input_map.csv\", index=False)\n    print(\"`test_img_input_map.csv` created and saved.\")\n    test_img_input_map.to_csv(\"./test_img_input_map.csv\", index=False)\n    print(\"`test_img_input_map.csv` created and saved.\")\nexcept Exception as e:\n    error_message = traceback.format_exc()\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_error.txt', 'w') as f:\n        f.write(error_message)\n        raise e","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:10.371197Z","iopub.execute_input":"2025-03-27T13:45:10.371562Z","iopub.status.idle":"2025-03-27T13:45:10.381354Z","shell.execute_reply.started":"2025-03-27T13:45:10.371535Z","shell.execute_reply":"2025-03-27T13:45:10.380519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\ntry:\n    import os\n    import pandas as pd\n    \n    # Define the base directory\n    base_dir = '/kaggle/input/cassava-leaf-disease-classification'\n    \n    # Read the train.csv file\n    train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))\n    \n    # Ensure the image_id column is renamed to 'id'\n    train_df.rename(columns={'image_id': 'id'}, inplace=True)\n    \n    # Rename the label column to indicate it is a classification target\n    train_df.rename(columns={'label': 'label_classification'}, inplace=True)\n    \n    # Create the final DataFrame\n    train_tab_target_map = train_df[['id', 'label_classification']]\n    \n    # <｜fim▁end｜>\n    # save\n    train_tab_target_map.to_csv(\"./train_tab_target_map.csv\", index=False)\n    print(\"`train_tab_target_map.csv` created and saved.\")\n    \n    \n    # <｜fim▁end｜>\n    # save\n    train_tab_target_map.to_csv(\"./train_tab_target_map.csv\", index=False)\n    print(\"`train_tab_target_map.csv` created and saved.\")\nexcept Exception as e:\n    error_message = traceback.format_exc()\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_error.txt', 'w') as f:\n        f.write(error_message)\n        raise e\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:12.600177Z","iopub.execute_input":"2025-03-27T13:45:12.600498Z","iopub.status.idle":"2025-03-27T13:45:12.652918Z","shell.execute_reply.started":"2025-03-27T13:45:12.600474Z","shell.execute_reply":"2025-03-27T13:45:12.652114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\n\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\n\n\n\ntry:\n    import os\n    import pandas as pd\n    base_dir = '/kaggle/input/cassava-leaf-disease-classification'\n    train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))\n    train_df.rename(columns={'image_id': 'id'}, inplace=True)\n    train_df['img_input1'] = train_df['id'].apply(lambda x: os.path.join(base_dir, 'train_images', x))\n    train_img_input_map = train_df[['id', 'img_input1']]\n    train_img_input_map.to_csv(\"./train_img_input_map.csv\", index=False)\n    print(\"`train_img_input_map.csv` created and saved.\")\n    train_img_input_map.to_csv(\"./train_img_input_map.csv\", index=False)\n    print(\"`train_img_input_map.csv` created and saved.\")\nexcept Exception as e:\n    error_message = traceback.format_exc()\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_error.txt', 'w') as f:\n        f.write(error_message)\n        raise e","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:14.645138Z","iopub.execute_input":"2025-03-27T13:45:14.645448Z","iopub.status.idle":"2025-03-27T13:45:14.788227Z","shell.execute_reply.started":"2025-03-27T13:45:14.645425Z","shell.execute_reply":"2025-03-27T13:45:14.787346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import atexit\nimport json\nimport os\nimport pathlib\nimport shutil\nimport time\nimport traceback\nimport warnings\nfrom argparse import ArgumentParser\nfrom functools import partial\nimport pytorch_lightning as L\nimport numpy as np\nimport pandas as pd\nimport torch.distributed as dist\nimport torch.optim\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom torch import nn, Tensor\nfrom torch.utils.data import DataLoader, WeightedRandomSampler\nfrom tqdm import tqdm\ntorch.manual_seed(123)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:16.070778Z","iopub.execute_input":"2025-03-27T13:45:16.071121Z","iopub.status.idle":"2025-03-27T13:45:16.079653Z","shell.execute_reply.started":"2025-03-27T13:45:16.071090Z","shell.execute_reply":"2025-03-27T13:45:16.078803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch.optim\nTAB_EMBED_LR = 1e-4\nTAB_HEAD_LR = 1e-4\nIMG_EMBED_LR = 1e-5\nIMG_HEAD_LR = 1e-5\nTXT_EMBED_LR = 1e-5\nTXT_HEAD_LR = 1e-5\nTRAIN_BATCH_SIZE = 32\nTEST_BATCH_SIZE = 32\nNUM_WORKERS = 4\nOPTIMIZER = torch.optim.Adam\nN_TRIALS = 20\nTTA_ROUNDS = 4\nif os.getenv(\"AGENT_DEBUG\"):\n    val_proportion = 0.05\nelse:\n    val_proportion = 0.25 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:17.903310Z","iopub.execute_input":"2025-03-27T13:45:17.903626Z","iopub.status.idle":"2025-03-27T13:45:17.908404Z","shell.execute_reply.started":"2025-03-27T13:45:17.903604Z","shell.execute_reply":"2025-03-27T13:45:17.907597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torchvision.models as models\nfrom torch import nn\nimport pandas as pd\nimport os\nfrom shutil import copyfile\n\nimport torch\nimport torchvision.models as models\nfrom torch import nn\ncopyfile(src = \"/kaggle/input/imgembedcorrect/img_embed.py\", dst = \"../working/img_embed.py\")\nfrom img_embed import ImageEmbedder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:19.799719Z","iopub.execute_input":"2025-03-27T13:45:19.800055Z","iopub.status.idle":"2025-03-27T13:45:19.807075Z","shell.execute_reply.started":"2025-03-27T13:45:19.800026Z","shell.execute_reply":"2025-03-27T13:45:19.806192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom torch import nn\nimport os.path\nimport pandas as pd\nimport torch\nfrom torch import nn\n# --- Design torch model and implement a differentiable torch loss functions\nclass TabularHead(nn.Module):\n    def __init__(self, embed_dim: int, output_dim: int):\n        super(TabularHead, self).__init__()\n        self.fc1 = nn.Linear(embed_dim, 256)\n        self.bn1 = nn.BatchNorm1d(256)\n        self.dropout1 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(256, 128)\n        self.bn2 = nn.BatchNorm1d(128)\n        self.dropout2 = nn.Dropout(0.5)\n        self.fc3 = nn.Linear(128, output_dim)\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = torch.relu(self.bn1(self.fc1(x)))\n        x = self.dropout1(x)\n        x = torch.relu(self.bn2(self.fc2(x)))\n        x = self.dropout2(x)\n        x = self.fc3(x)\n        return x\ndef regression_loss(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n    return torch.nn.functional.mse_loss(pred, target)\ndef classification_loss(pred_logits: torch.Tensor, target_one_hot: torch.Tensor) -> torch.Tensor:\n    return torch.nn.functional.cross_entropy(pred_logits, target_one_hot.argmax(dim=1))\n    # --- Test losses\n    # For regression targets:\n    target = torch.rand(batch_size, output_dim, dtype=torch.float)\n    reg_loss = regression_loss(output, target).mean()\n    # For classification targets:\n    target = torch.rand(batch_size, output_dim, dtype=torch.float)\n    class_loss = classification_loss(output, target).mean()\n    # @NO_MEMORY_START@\n    print(f\"Could compute tabular outputs and losses without error.\")\n    print(f\"Output size: {output.shape[-1]}\")\n    # @NO_MEMORY_END@\nclass SubmissionFormatError(Exception):\n    \"\"\"When all attempts to create the submission format have failed\"\"\"\n    pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:30.164637Z","iopub.execute_input":"2025-03-27T13:45:30.164933Z","iopub.status.idle":"2025-03-27T13:45:30.173226Z","shell.execute_reply.started":"2025-03-27T13:45:30.164910Z","shell.execute_reply":"2025-03-27T13:45:30.172334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"copyfile(src=\"/kaggle/input/mapdataset/map_dataset.py\", dst=\"./map_dataset.py\")\nfrom map_dataset import Identity, MapDataset, map_dataset_collate_function\ntab_fe_preprocess = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:32.572727Z","iopub.execute_input":"2025-03-27T13:45:32.573025Z","iopub.status.idle":"2025-03-27T13:45:32.580333Z","shell.execute_reply.started":"2025-03-27T13:45:32.572992Z","shell.execute_reply":"2025-03-27T13:45:32.579372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\n\nimport pandas as pd\n\ndef calculate_class_weights(target_df:pd.DataFrame, target_columns: list) -> np.ndarray:\n    # Generate a weight vector of ones (same length as the number of samples)\n    weight_vector = np.ones(len(target_df))\n\n    return weight_vector\n\nclass_weights_fn = calculate_class_weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:33.404954Z","iopub.execute_input":"2025-03-27T13:45:33.405256Z","iopub.status.idle":"2025-03-27T13:45:33.409510Z","shell.execute_reply.started":"2025-03-27T13:45:33.405233Z","shell.execute_reply":"2025-03-27T13:45:33.408596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nThis script creates the transform functions for the tabular training targets.\n\"\"\"\nimport json\nfrom typing import Any, Union, Tuple\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.preprocessing import OneHotEncoder\n\n# Read target tab file\ndf_train_target = pd.read_csv('./train_tab_target_map.csv')\n\n# Create onehot encoder for transforming the target\nclass_names_columns_regression = [col for col in df_train_target.columns if col.endswith('_regression')]\nclass_names_columns_classification = [col for col in df_train_target.columns if col.endswith('_classification')]\nclass_names_data_classification = df_train_target[class_names_columns_classification].values\nif len(class_names_columns_classification) > 0:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(class_names_data_classification)\n    for col, cat in zip(class_names_columns_classification, enc.categories_):\n        if len(cat) > 0.5 * class_names_data_classification.shape[0]:\n            raise ValueError(f\"Trying to one-hot encode {len(class_names_columns_classification)} classification \"\n                             f\"columns, but at least one column has {len(cat)} distinct values. Are you sure \"\n                             f\"the column {col} is a categorical column?\")\nelse:\n    enc = None\n\n\ntarget_columns_transform = target_tranform\n\n\ndef tab_target_transform(original_target: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Transform for tabular targets, maps original submission format to usable numerical format.\n    It cannot return `None`, it has to at least return `original_target` if no transform is needed.\n    \"\"\"\n\n    # Filter regression columns\n    df_regression = original_target.filter(like='_regression', axis=1)\n    if enc is None:\n        df_regression.insert(loc=0, column='id', value=original_target['id'].values)\n        return df_regression\n\n    # Convert class labels to one-hot encodings\n    class_name = list(enc.get_feature_names_out(class_names_columns_classification))\n    x = original_target[class_names_columns_classification].values\n    onehot_class = enc.transform(x)\n\n    df_classification = pd.DataFrame([], columns=['id'] + class_name)\n    df_classification['id'] = original_target['id'].values\n    df_classification.iloc[:, 1:] = onehot_class.toarray()\n\n    # Concatenate the two dataframe\n    transformed_target = pd.concat([df_classification, df_regression],\n                                   axis=1) if not df_regression.empty else df_classification\n    transformed_target = transformed_target.infer_objects()\n    return transformed_target\n\n\ndef onehot_to_classname(\n        onehot: np.array,\n        probabilities: np.array,\n        classification_threshold: float,\n        unknown_classname: str | None\n):\n    \"\"\"\n    This function converts a one-hot encoded array back to class names, and replaces classes with probabilities below a\n    threshold with a specified 'unknown' class name.\n\n    Parameters:\n    onehot (np.array): A one-hot encoded numpy array representing class memberships.\n    probabilities (np.array): A numpy array of class probabilities corresponding to the classes in 'onehot'.\n    classification_threshold (float): A threshold for class probabilities. Classes with probabilities below this threshold are considered 'unknown'.\n    unknown_classname (str): A string to replace the class names of 'unknown' classes (i.e., classes with probabilities below the threshold).\n\n    Returns:\n    class_names_array (np.array): A numpy array of class names. 'Unknown' classes have been replaced with 'unknown_classname'.\n    \"\"\"\n\n    class_names_array = enc.inverse_transform(onehot)\n    class_names_array = class_names_array[class_names_array != np.array(None)]\n\n    if unknown_classname:\n        unknown_classes_idx = np.argwhere(probabilities < classification_threshold)\n        class_names_array = class_names_array.astype(object)\n        class_names_array[unknown_classes_idx] = unknown_classname\n\n    return class_names_array\n\n\ndef extract_values_top_k(input_string: str) -> Tuple[int, str]:\n    \"\"\"\n    Extracts values between parentheses and the comma.\n\n    Args:\n        input_string (str): A string containing values in the format '(x, y)'.\n\n    Returns:\n        list: A list containing the extracted values.\n\n    Example:\n        extract_values('(5, new_whale)')\n        ['5', 'new_whale']\n    \"\"\"\n    result = re.findall(r'[^,()]+', input_string)\n    top_k = int(result[0])\n    unknown_class_name = result[1].replace(' ', '')\n    unknown_class_name = None if 'none' in unknown_class_name.lower() else unknown_class_name\n    return top_k, unknown_class_name\n\n\ndef process_classification_target(\n        classification_probits: np.array,\n        columns_target_map: dict,\n        columns_class_name: list,\n        classification_threshold: float = 1e-5\n\n) -> pd.DataFrame:\n    \"\"\"\n    This function processes a classification target by grouping labels, extracting features, and creating a DataFrame.\n\n    Parameters:\n    classification_target (np.array): A numpy array representing the classification probabilities.\n    columns_target_map (dict): A dictionary mapping column names to targets.\n    columns_class_name (list): A list of class names for the columns.\n    classification_threshold (float, optional): A threshold for classification. Default is 1e-5.\n\n    Returns:\n    df_classification (pd.DataFrame): A pandas DataFrame that contains the processed classification target.\n    \"\"\"\n    groups_label = enc.categories_\n    idx_start = 0\n    df_classification = pd.DataFrame()\n\n    for group in groups_label:\n\n        # Extract the label columns names and probits of the current classification group\n        label_cols_names = columns_class_name[idx_start:idx_start + len(group)]\n        group_probits = classification_probits[:, idx_start:idx_start + len(group)]\n\n        # check that probits are indeed probits\n        assert np.isclose(group_probits.sum(axis=-1), 1.0).all(), \\\n            f\"Probits of group {group} do not sum to 1! Did you forget to apply softmax?\"\n\n        # Extract the feature column name\n        feature_column_name = label_cols_names[0][:label_cols_names[0].rfind(\"_classification\")] + \"_classification\"\n\n        if columns_target_map[feature_column_name] == \"proba\":\n            df = pd.DataFrame(group_probits, columns=label_cols_names)\n\n        else:\n            raw_top_k = columns_target_map[feature_column_name].split('top_k')[1]\n            top_k, unknown_classname = extract_values_top_k(input_string=raw_top_k)\n            onehot_group, proba_group = get_topk_onehot(probits=group_probits, k=top_k)\n\n            # Format the onot hot to match shape of the onehot encoder\n            onehot = np.zeros((classification_probits.shape[0] * top_k, classification_probits.shape[1]))\n            onehot[:, idx_start:idx_start + len(group)] = onehot_group\n\n            class_names_array = onehot_to_classname(\n                onehot=onehot,\n                probabilities=proba_group,\n                classification_threshold=classification_threshold,\n                unknown_classname=unknown_classname,\n            )\n\n            class_names_array = class_names_array.reshape(classification_probits.shape[0], top_k)\n\n            if top_k > 1:\n                class_names_array = pd.Series(list(class_names_array))\n\n            df = pd.DataFrame(class_names_array, columns=[feature_column_name])\n\n        df_classification = pd.concat([df_classification, df], axis=1)\n\n        idx_start += len(group)\n\n    return df_classification\n\n\ndef get_topk_onehot(\n        probits: np.array,\n        k: int = 1,\n) -> np.array:\n    \"\"\"\n    This function returns the top 'k' values and their indices from the input array 'probits'.\n\n    Parameters:\n    probits (np.array): A numpy array from which to select the top 'k' values.\n    k (int, optional): The number of top values to select. Default is 1.\n\n    Returns:\n    onehot (np.array): A one-hot encoded numpy array of shape (probits.shape[0] * k, probits.shape[1]).\n                       The 'i'th row of 'onehot' corresponds to the 'i'th top value in 'probits'.\n    values (np.array): A flattened numpy array of the top 'k' values in 'probits'.\n    \"\"\"\n    values, indices = torch.topk(torch.Tensor(probits), k)\n    onehot = np.zeros((probits.shape[0] * k, probits.shape[1]))\n    onehot[np.arange(onehot.shape[0]), indices.flatten().numpy()] = 1\n\n    return onehot, values.flatten().numpy()\n\n\ndef tab_target_inverse_transform(target_values: np.ndarray, ids: Union[list[Any], np.array]) -> pd.DataFrame:\n    \"\"\"\n    Inverse transform for tabular targets, maps back to original submission format.\n    It cannot return `None`, it has to at least return the `transformed_target` if no inverse transform is needed.\n    Args:\n        target_values: batch of target values\n        ids: list of the entry ids\n    \"\"\"\n    if enc is None:\n        regression_target = target_values\n    else:\n        columns_class_name = list(enc.get_feature_names_out(class_names_columns_classification))\n        classification_target = target_values[:, :len(columns_class_name)]\n        regression_target = target_values[:, len(columns_class_name):]\n\n    df_regression_target = pd.DataFrame(regression_target, columns=class_names_columns_regression)\n    if enc is None:\n        df_regression_target.insert(0, \"id\", ids)\n        return df_regression_target\n\n    # We assume that if we are here, it's because at least one target column is a classification,\n    # otherwise, if the task is purely a classification task, we probably shouldn't end up here ...\n    df_transformed_classification_target = process_classification_target(\n        classification_probits=classification_target,\n        columns_target_map=target_columns_transform,\n        columns_class_name=columns_class_name,\n    )\n\n    transformed_target = pd.concat([df_transformed_classification_target, df_regression_target], axis=1)\n    transformed_target.insert(0, \"id\", ids)\n\n    for pattern in [\"_regression\", \"_classification\"]:\n        transformed_target.rename({c: c.replace(pattern, \"\") for c in transformed_target.columns}, axis=1, inplace=True)\n    return transformed_target\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:35.041222Z","iopub.execute_input":"2025-03-27T13:45:35.041572Z","iopub.status.idle":"2025-03-27T13:45:35.073315Z","shell.execute_reply.started":"2025-03-27T13:45:35.041543Z","shell.execute_reply":"2025-03-27T13:45:35.072589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ncustom_tab_regression_scaler = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:39.621719Z","iopub.execute_input":"2025-03-27T13:45:39.622031Z","iopub.status.idle":"2025-03-27T13:45:39.625878Z","shell.execute_reply.started":"2025-03-27T13:45:39.622007Z","shell.execute_reply":"2025-03-27T13:45:39.624781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimg_target_transform = Identity()\nimg_target_inverse_transform = Identity()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:40.217674Z","iopub.execute_input":"2025-03-27T13:45:40.217988Z","iopub.status.idle":"2025-03-27T13:45:40.221606Z","shell.execute_reply.started":"2025-03-27T13:45:40.217961Z","shell.execute_reply":"2025-03-27T13:45:40.220727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntxt_target_transform = Identity()\ntxt_target_inverse_transform = Identity()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:40.841938Z","iopub.execute_input":"2025-03-27T13:45:40.842216Z","iopub.status.idle":"2025-03-27T13:45:40.846214Z","shell.execute_reply.started":"2025-03-27T13:45:40.842195Z","shell.execute_reply":"2025-03-27T13:45:40.845310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch import nn\nfrom PIL import Image\nimport torchvision.transforms as T\n\n\nimport torchvision.transforms as T\n\nCustomTrainImageInputTransform: T.Compose = T.Compose([\n    T.Resize((256, 256)),\n    T.RandomHorizontalFlip(p=0.5),\n    T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    T.RandomCrop(224),\n    T.ToTensor(),\n    T.Normalize(mean=[0.43032029271125793, 0.4967266917228699, 0.31341689825057983],\n                std=[0.23058539628982544, 0.2335016131401062, 0.22131390869617462]),\n    T.Lambda(lambda x: x[:3] if x.shape[0] > 3 else x)\n])\n\n\n\nimport torchvision.transforms as T\n\nCustomTestImageInputTransform: T.Compose = T.Compose([\n    T.Resize((256, 256)),\n    T.CenterCrop(224),\n    T.ToTensor(),\n    T.Normalize(mean=[0.43032029271125793, 0.4967266917228699, 0.31341689825057983],\n                std=[0.23058539628982544, 0.2335016131401062, 0.22131390869617462]),\n    T.Lambda(lambda x: x[:3] if x.shape[0] > 3 else x)\n])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:42.156851Z","iopub.execute_input":"2025-03-27T13:45:42.157152Z","iopub.status.idle":"2025-03-27T13:45:42.163697Z","shell.execute_reply.started":"2025-03-27T13:45:42.157127Z","shell.execute_reply":"2025-03-27T13:45:42.162826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_format_functions = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:44.241009Z","iopub.execute_input":"2025-03-27T13:45:44.241312Z","iopub.status.idle":"2025-03-27T13:45:44.244888Z","shell.execute_reply.started":"2025-03-27T13:45:44.241288Z","shell.execute_reply":"2025-03-27T13:45:44.244029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_names = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:44.851855Z","iopub.execute_input":"2025-03-27T13:45:44.852154Z","iopub.status.idle":"2025-03-27T13:45:44.855672Z","shell.execute_reply.started":"2025-03-27T13:45:44.852131Z","shell.execute_reply":"2025-03-27T13:45:44.854703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\n\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\n\n\n\n\n\"\"\"\nThis script creates a function that takes an input DataFrame and formats it to follow the df_sample_submission format\n\"\"\"\n# useful imports\nimport os\nimport numpy as np\nimport pandas as pd\n\n\n### Submission format definition\ndef df_to_submission_format(\n    input_df: pd.DataFrame,\n) -> pd.DataFrame:\n    \"\"\"\n    Format the input_df to the submission format DataFrame\n    Args:\n        input_df: DataFrame to be formatted\n    Return:\n        format_submission_df: DataFrame formatted\n    \"\"\"\n    # Rename the 'id' column to 'image_id'\n    input_df = input_df.rename(columns={'id': 'image_id'})\n    \n    # Ensure the 'label' column contains only integer values\n    if not np.issubdtype(input_df['label'].dtype, np.integer):\n        input_df['label'] = input_df['label'].astype(int)\n    \n    # Check if the 'label' column contains only allowed values\n    allowed_values = [0, 1, 2, 3, 4]\n    if not input_df['label'].isin(allowed_values).all():\n        raise ValueError(\"The 'label' column contains values outside the allowed range [0, 1, 2, 3, 4].\")\n    \n    # Return the formatted DataFrame\n    return input_df\n\nsubmission_format_functions.append(df_to_submission_format)\nsubmission_names.append('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:45.513589Z","iopub.execute_input":"2025-03-27T13:45:45.513887Z","iopub.status.idle":"2025-03-27T13:45:45.520040Z","shell.execute_reply.started":"2025-03-27T13:45:45.513863Z","shell.execute_reply":"2025-03-27T13:45:45.519026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\n\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\n\n\n\n\n\"\"\"\nThis script creates a function that takes an input Dataframe and format it to follow the df_sample_submission format\n\"\"\"\n# useful imports\nimport os\nimport numpy as np\nimport pandas as pd\n\n\n### Submission format definition\ndef df_to_submission_format_alt(\n    input_df: pd.DataFrame,\n) -> pd.DataFrame:\n    \"\"\"\n    Format the input_df to the submission format DataFrame\n    Args:\n        input_df: DataFrame to be formatted\n    Return:\n        format_submission_df: DataFrame formatted\n    \"\"\"\n    # Step 1: Rename the 'id' column to 'image_id'\n    input_df = input_df.rename(columns={'id': 'image_id'})\n    \n    # Step 2: Ensure the 'label' column contains integer values\n    input_df['label'] = input_df['label'].astype(int)\n    \n    # Step 3: Reorder the columns to match the required format\n    format_submission_df = input_df[['image_id', 'label']]\n    \n    return format_submission_df\n\nsubmission_format_functions.append(df_to_submission_format_alt)\nsubmission_names.append('submission_alt.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:48.317592Z","iopub.execute_input":"2025-03-27T13:45:48.317888Z","iopub.status.idle":"2025-03-27T13:45:48.323634Z","shell.execute_reply.started":"2025-03-27T13:45:48.317865Z","shell.execute_reply":"2025-03-27T13:45:48.322661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if len(submission_format_functions) == 0:\n    raise ValueError('No valid submission format functions found in setup!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:49.155604Z","iopub.execute_input":"2025-03-27T13:45:49.155899Z","iopub.status.idle":"2025-03-27T13:45:49.159766Z","shell.execute_reply.started":"2025-03-27T13:45:49.155875Z","shell.execute_reply":"2025-03-27T13:45:49.158832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport traceback\nimport warnings\n\ndef write_warning_to_file(message, category, filename, lineno, file=None, line=None):\n    with open('/nfs/aiml/alexandre/Projects/agent/workspace/cassava-leaf-disease-classification/seed_0/_code_warnings.txt', 'w') as f:\n        f.write(warnings.formatwarning(message=message, category=category, filename=filename, lineno=lineno, line=line))\n\n\n\n\n\"\"\"\nThis script creates a function that takes the predicted output `y_pred` and the true output `y_true`\nand returns the value of the metric corresponding to the task.\n\"\"\"\n# useful imports\nimport os\nimport numpy as np\nimport pandas as pd\nfrom torch import Tensor\n\n# metric definition\ndef metric_function(\n        y_pred: pd.DataFrame | Tensor,\n        y_true: pd.DataFrame | Tensor,\n) -> float:\n    \"\"\"\n    Computes the metric on (a batch of) inputs and returns the result.\n    Args:\n        y_pred: the predicted target\n        y_true: the true target\n    \"\"\"\n    # Ensure both inputs are pandas DataFrames\n    if isinstance(y_pred, Tensor):\n        y_pred = pd.DataFrame({'image_id': y_pred[:, 0].numpy(), 'label': y_pred[:, 1].numpy()})\n    if isinstance(y_true, Tensor):\n        y_true = pd.DataFrame({'image_id': y_true[:, 0].numpy(), 'label': y_true[:, 1].numpy()})\n\n    # Drop the 'image_id' column to focus on the labels\n    y_pred_labels = y_pred['label']\n    y_true_labels = y_true['label']\n\n    # Ensure the lengths of y_pred and y_true are the same\n    if len(y_pred_labels) != len(y_true_labels):\n        raise ValueError(\"The length of y_pred and y_true must be the same.\")\n\n    # Count the number of correct predictions\n    correct_predictions = (y_pred_labels == y_true_labels).sum()\n\n    # Calculate the accuracy\n    accuracy = correct_predictions / len(y_true_labels)\n\n    return accuracy\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:51.343019Z","iopub.execute_input":"2025-03-27T13:45:51.343359Z","iopub.status.idle":"2025-03-27T13:45:51.349967Z","shell.execute_reply.started":"2025-03-27T13:45:51.343327Z","shell.execute_reply":"2025-03-27T13:45:51.349003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NanLossError(Exception):\n    \"\"\" raise when there is a nan loss\"\"\"\n    pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:54.049608Z","iopub.execute_input":"2025-03-27T13:45:54.049897Z","iopub.status.idle":"2025-03-27T13:45:54.053610Z","shell.execute_reply.started":"2025-03-27T13:45:54.049875Z","shell.execute_reply":"2025-03-27T13:45:54.052745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optional_path(path: str) -> str | None:\n    \"\"\"Return path if it exists, else return None\"\"\"\n    if os.path.exists(path):\n        return path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:54.870874Z","iopub.execute_input":"2025-03-27T13:45:54.871161Z","iopub.status.idle":"2025-03-27T13:45:54.875216Z","shell.execute_reply.started":"2025-03-27T13:45:54.871139Z","shell.execute_reply":"2025-03-27T13:45:54.874184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tab_input_map_path = get_optional_path('train_tab_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:45:55.769322Z","iopub.execute_input":"2025-03-27T13:45:55.769649Z","iopub.status.idle":"2025-03-27T13:45:55.773325Z","shell.execute_reply.started":"2025-03-27T13:45:55.769621Z","shell.execute_reply":"2025-03-27T13:45:55.772355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_input_map_path = get_optional_path('train_img_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:02.982482Z","iopub.execute_input":"2025-03-27T13:46:02.982786Z","iopub.status.idle":"2025-03-27T13:46:02.986963Z","shell.execute_reply.started":"2025-03-27T13:46:02.982763Z","shell.execute_reply":"2025-03-27T13:46:02.985868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"txt_input_map_path = get_optional_path('train_txt_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:04.544080Z","iopub.execute_input":"2025-03-27T13:46:04.544449Z","iopub.status.idle":"2025-03-27T13:46:04.548750Z","shell.execute_reply.started":"2025-03-27T13:46:04.544418Z","shell.execute_reply":"2025-03-27T13:46:04.547692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tab_target_map_path = get_optional_path('train_tab_target_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:04.754571Z","iopub.execute_input":"2025-03-27T13:46:04.754781Z","iopub.status.idle":"2025-03-27T13:46:04.758195Z","shell.execute_reply.started":"2025-03-27T13:46:04.754762Z","shell.execute_reply":"2025-03-27T13:46:04.757255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_target_map_path = get_optional_path('train_img_target_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:04.917687Z","iopub.execute_input":"2025-03-27T13:46:04.917894Z","iopub.status.idle":"2025-03-27T13:46:04.921410Z","shell.execute_reply.started":"2025-03-27T13:46:04.917875Z","shell.execute_reply":"2025-03-27T13:46:04.920480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"txt_target_map_path = get_optional_path('train_txt_target_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:05.102533Z","iopub.execute_input":"2025-03-27T13:46:05.102761Z","iopub.status.idle":"2025-03-27T13:46:05.106117Z","shell.execute_reply.started":"2025-03-27T13:46:05.102730Z","shell.execute_reply":"2025-03-27T13:46:05.105321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_tab_input_map_path = get_optional_path('test_tab_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:05.279550Z","iopub.execute_input":"2025-03-27T13:46:05.279765Z","iopub.status.idle":"2025-03-27T13:46:05.283330Z","shell.execute_reply.started":"2025-03-27T13:46:05.279738Z","shell.execute_reply":"2025-03-27T13:46:05.282353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_img_input_map_path = get_optional_path('test_img_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:05.467133Z","iopub.execute_input":"2025-03-27T13:46:05.467374Z","iopub.status.idle":"2025-03-27T13:46:05.470523Z","shell.execute_reply.started":"2025-03-27T13:46:05.467354Z","shell.execute_reply":"2025-03-27T13:46:05.469647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_txt_input_map_path = get_optional_path('test_txt_input_map.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:07.241239Z","iopub.execute_input":"2025-03-27T13:46:07.241545Z","iopub.status.idle":"2025-03-27T13:46:07.245297Z","shell.execute_reply.started":"2025-03-27T13:46:07.241522Z","shell.execute_reply":"2025-03-27T13:46:07.244327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(dataset, test_dataset) = MapDataset.create_train_test_datasets(train_tab_input_map_path=tab_input_map_path, train_img_input_map_path=img_input_map_path, train_txt_input_map_path=txt_input_map_path, train_tab_target_map_path=tab_target_map_path, train_img_target_map_path=img_target_map_path, train_txt_target_map_path=txt_target_map_path, test_tab_input_map_path=test_tab_input_map_path, test_img_input_map_path=test_img_input_map_path, test_txt_input_map_path=test_txt_input_map_path, tab_input_transform=Identity(), img_input_transform=Identity(), txt_input_transform=Identity(), tab_target_transform=tab_target_transform, img_target_transform=img_target_transform, txt_target_transform=txt_target_transform, tab_target_inverse_transform=tab_target_inverse_transform, img_target_inverse_transform=img_target_inverse_transform, txt_target_inverse_transform=txt_target_inverse_transform, custom_tab_regression_scaler=custom_tab_regression_scaler, custom_img_train_input_transform=CustomTrainImageInputTransform, custom_img_test_input_transform=CustomTestImageInputTransform, tab_fe=tab_fe_preprocess.preprocess if tab_fe_preprocess else None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:07.440970Z","iopub.execute_input":"2025-03-27T13:46:07.441213Z","iopub.status.idle":"2025-03-27T13:46:07.545326Z","shell.execute_reply.started":"2025-03-27T13:46:07.441191Z","shell.execute_reply":"2025-03-27T13:46:07.544466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.getenv('AGENT_DEBUG'):\n    val_proportion = 0.05\nelse:\n    val_proportion = 0.25","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:07.625258Z","iopub.execute_input":"2025-03-27T13:46:07.625507Z","iopub.status.idle":"2025-03-27T13:46:07.629330Z","shell.execute_reply.started":"2025-03-27T13:46:07.625487Z","shell.execute_reply":"2025-03-27T13:46:07.628468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train_dataset, validation_dataset) = dataset.split(frac=val_proportion)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:07.864394Z","iopub.execute_input":"2025-03-27T13:46:07.864615Z","iopub.status.idle":"2025-03-27T13:46:07.965157Z","shell.execute_reply.started":"2025-03-27T13:46:07.864596Z","shell.execute_reply":"2025-03-27T13:46:07.964548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_dataset.img_input_transform = test_dataset.img_input_transform","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:08.022125Z","iopub.execute_input":"2025-03-27T13:46:08.022381Z","iopub.status.idle":"2025-03-27T13:46:08.025794Z","shell.execute_reply.started":"2025-03-27T13:46:08.022360Z","shell.execute_reply":"2025-03-27T13:46:08.024822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_data_loaders(train_batch_size: int=TRAIN_BATCH_SIZE, val_batch_size: int=TEST_BATCH_SIZE):\n    if class_weights_fn is not None:\n        target_df = train_dataset.tab_target_map.copy(deep=True)\n        target_df = target_df.drop(columns='id', errors='ignore')\n        target_columns = target_df.columns.tolist()\n        weight_vector = calculate_class_weights(target_df=target_df, target_columns=target_columns)\n        weight_vector = torch.from_numpy(weight_vector.astype(np.float32))\n        sampler = WeightedRandomSampler(weight_vector, len(weight_vector))\n        shuffle = False\n    else:\n        sampler = None\n        shuffle = True\n    train_dl = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=shuffle, pin_memory=True, prefetch_factor=2, persistent_workers=True, collate_fn=map_dataset_collate_function, num_workers=NUM_WORKERS, sampler=sampler)\n    valid_dl = DataLoader(validation_dataset, batch_size=val_batch_size, shuffle=False, pin_memory=True, prefetch_factor=2, collate_fn=map_dataset_collate_function, num_workers=NUM_WORKERS)\n    test_dl = DataLoader(test_dataset, batch_size=val_batch_size, shuffle=False, collate_fn=map_dataset_collate_function, num_workers=NUM_WORKERS)\n    return (train_dl, valid_dl, test_dl)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:08.922515Z","iopub.execute_input":"2025-03-27T13:46:08.922898Z","iopub.status.idle":"2025-03-27T13:46:08.930881Z","shell.execute_reply.started":"2025-03-27T13:46:08.922864Z","shell.execute_reply":"2025-03-27T13:46:08.929835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train_dataloader, validation_dataloader, test_dataloader) = get_data_loaders(TRAIN_BATCH_SIZE, TEST_BATCH_SIZE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:09.141412Z","iopub.execute_input":"2025-03-27T13:46:09.141702Z","iopub.status.idle":"2025-03-27T13:46:09.185180Z","shell.execute_reply.started":"2025-03-27T13:46:09.141676Z","shell.execute_reply":"2025-03-27T13:46:09.184404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_input_sample = next(iter(train_dataloader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-27T13:46:09.297025Z","iopub.execute_input":"2025-03-27T13:46:09.297238Z","iopub.status.idle":"2025-03-27T13:46:11.697490Z","shell.execute_reply.started":"2025-03-27T13:46:09.297220Z","shell.execute_reply":"2025-03-27T13:46:11.695233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(tab_inputs_batch, img_inputs_batch, txt_inputs_batch) = (None, None, None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:43.343166Z","iopub.execute_input":"2025-03-11T15:56:43.343555Z","iopub.status.idle":"2025-03-11T15:56:43.350137Z","shell.execute_reply.started":"2025-03-11T15:56:43.343503Z","shell.execute_reply":"2025-03-11T15:56:43.348871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(tab_targets_batch, img_targets_batch, txt_targets_batch) = (None, None, None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:43.351806Z","iopub.execute_input":"2025-03-11T15:56:43.352412Z","iopub.status.idle":"2025-03-11T15:56:43.373639Z","shell.execute_reply.started":"2025-03-11T15:56:43.352374Z","shell.execute_reply":"2025-03-11T15:56:43.372561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for (indices, (tab_inputs_batch, img_inputs_batch, txt_inputs_batch), (tab_targets_batch, img_targets_batch, txt_targets_batch)) in train_dataloader:\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:43.374704Z","iopub.execute_input":"2025-03-11T15:56:43.375298Z","iopub.status.idle":"2025-03-11T15:56:46.814743Z","shell.execute_reply.started":"2025-03-11T15:56:43.375262Z","shell.execute_reply":"2025-03-11T15:56:46.813876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if tab_targets_batch is not None:\n    OUTPUT_DIM = tab_targets_batch.shape[1]\nelif img_targets_batch is not None:\n    OUTPUT_DIM = img_targets_batch.shape[1]\nelif txt_targets_batch is not None:\n    OUTPUT_DIM = txt_targets_batch.shape[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:46.815454Z","iopub.execute_input":"2025-03-11T15:56:46.815729Z","iopub.status.idle":"2025-03-11T15:56:46.827649Z","shell.execute_reply.started":"2025-03-11T15:56:46.815701Z","shell.execute_reply":"2025-03-11T15:56:46.826616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_tab_embedder():\n    if 'TabularEmbedder' in globals():\n        tab_input_dim = tab_inputs_batch.shape[-1]\n        tab_embedder = TabularEmbedder(input_dim=tab_input_dim, embed_dim=TAB_EMBED_DIM)\n        tab_embed_dim = TAB_EMBED_DIM\n    else:\n        tab_embedder = None\n        tab_embed_dim = 0\n    return (tab_embedder, tab_embed_dim)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:46.831526Z","iopub.execute_input":"2025-03-11T15:56:46.831838Z","iopub.status.idle":"2025-03-11T15:56:46.860648Z","shell.execute_reply.started":"2025-03-11T15:56:46.831809Z","shell.execute_reply":"2025-03-11T15:56:46.859575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_img_embedder():\n    if 'ImageEmbedder' in globals():\n        assert issubclass(ImageEmbedder, nn.Module)\n\n        class AuxImageEmbedder(ImageEmbedder):\n            \"\"\" Wrap ImageEmbedder to deal with the input dimension \"\"\"\n\n            def forward(self, x: torch.Tensor):\n                \"\"\"\n                Args:\n                    x: dimension (batch, n_images_per_id, n_channels, height, width)\n\n                Returns:\n                     y: dimension (batch, embed_dim)  --> the `n_images_per_id` are flatten\n                \"\"\"\n                y = super().forward(x.view(-1, *x.shape[-3:]))\n                return y.reshape(len(x), -1)\n        img_embedder = AuxImageEmbedder()\n        with torch.no_grad():\n            img_embed_dim = img_embedder(img_inputs_batch[:2]).shape[-1]\n    else:\n        img_embedder = None\n        img_embed_dim = 0\n    return (img_embedder, img_embed_dim)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:50.765673Z","iopub.execute_input":"2025-03-11T15:56:50.765990Z","iopub.status.idle":"2025-03-11T15:56:50.771644Z","shell.execute_reply.started":"2025-03-11T15:56:50.765965Z","shell.execute_reply":"2025-03-11T15:56:50.770685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_txt_embedder():\n    if 'TextEmbedder' in globals():\n        assert issubclass(TextEmbedder, nn.Module)\n\n        class AuxTextEmbedder(TextEmbedder):\n            \"\"\" Wrap TextEmbedder to deal with the input dimension \"\"\"\n\n            def forward(self, x: pd.DataFrame | np.ndarray | torch.Tensor):\n                \"\"\"\n                Args:\n                    x: dimension (batch, n_texts_per_id)\n\n                Returns:\n                     y: dimension (batch, embed_dim)  --> the `n_texts_per_id` are flattened\n                \"\"\"\n                bsz = x.shape[0]\n                if isinstance(x, pd.DataFrame):\n                    x = x.values\n                    x = x.reshape(-1, *x.shape[1:])\n                    x = x.flatten().tolist()\n                y = super(AuxTextEmbedder, self).forward(x)\n                return y.reshape(bsz, -1)\n        txt_embedder = AuxTextEmbedder()\n        with torch.no_grad():\n            txt_embed_dim = txt_embedder(txt_inputs_batch).shape[-1]\n    else:\n        txt_embedder = None\n        txt_embed_dim = 0\n    return (txt_embedder, txt_embed_dim)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:51.099282Z","iopub.execute_input":"2025-03-11T15:56:51.099637Z","iopub.status.idle":"2025-03-11T15:56:51.105306Z","shell.execute_reply.started":"2025-03-11T15:56:51.099608Z","shell.execute_reply":"2025-03-11T15:56:51.104448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_tab_head():\n    return (TabularHead, regression_loss, classification_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:51.452437Z","iopub.execute_input":"2025-03-11T15:56:51.452737Z","iopub.status.idle":"2025-03-11T15:56:51.456261Z","shell.execute_reply.started":"2025-03-11T15:56:51.452712Z","shell.execute_reply":"2025-03-11T15:56:51.455474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nget_img_head = None\nimg_loss = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:51.802770Z","iopub.execute_input":"2025-03-11T15:56:51.803012Z","iopub.status.idle":"2025-03-11T15:56:51.806508Z","shell.execute_reply.started":"2025-03-11T15:56:51.802993Z","shell.execute_reply":"2025-03-11T15:56:51.805738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nget_txt_head = None\ntxt_loss = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:52.137499Z","iopub.execute_input":"2025-03-11T15:56:52.137716Z","iopub.status.idle":"2025-03-11T15:56:52.141016Z","shell.execute_reply.started":"2025-03-11T15:56:52.137697Z","shell.execute_reply":"2025-03-11T15:56:52.140207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SubmissionModel(L.LightningModule):\n\n    def __init__(self, learning_rate=1e-05, optimizer_choice='adam'):\n        super().__init__()\n        (tab_embedder, tab_embed_dim) = get_tab_embedder()\n        (img_embedder, img_embed_dim) = get_img_embedder()\n        (txt_embedder, txt_embed_dim) = get_txt_embedder()\n        self.tab_embedder = tab_embedder\n        self.img_embedder = img_embedder\n        self.txt_embedder = txt_embedder\n        self.embed_dim = tab_embed_dim + img_embed_dim + txt_embed_dim\n        (TabularHead, regression_loss, classification_loss) = get_tab_head()\n        self.tab_head = TabularHead(embed_dim=self.embed_dim, output_dim=OUTPUT_DIM)\n        self.learning_rate = learning_rate\n        self.optimizer_choice = optimizer_choice\n        if get_img_head is None:\n            self.img_head = None\n        else:\n            self.img_head = get_img_head()\n        if get_txt_head is None:\n            self.txt_head = None\n        else:\n            self.txt_head = get_txt_head()\n        self.tab_regression_loss = regression_loss\n        self.tab_classification_loss = classification_loss\n        self.img_loss = img_loss\n        self.txt_loss = txt_loss\n        self.unfreeze_epoch = None\n\n    def tab_loss(self, pred: torch.Tensor, target: pd.DataFrame) -> torch.Tensor:\n        assert pred.shape == target.shape, (pred.shape, target.shape)\n        target = torch.from_numpy(target.values).to(pred)\n        if pred.ndim == 1:\n            pred = pred.unsqueeze(0)\n            target = target.unsqueeze(0)\n        loss = 0.0\n        if enc:\n            columns_class_name = list(enc.get_feature_names_out(class_names_columns_classification))\n            classification_pred = pred[:, :len(columns_class_name)]\n            classification_target = target[:, :len(columns_class_name)]\n            groups_label = enc.categories_\n            idx_start = 0\n            for group in groups_label:\n                group_pred = classification_pred[:, idx_start:idx_start + len(group)]\n                group_target = classification_target[:, idx_start:idx_start + len(group)]\n                idx_start += len(group)\n                loss += self.tab_classification_loss(group_pred, group_target).mean()\n            if pred.shape[-1] > len(columns_class_name):\n                regression_pred = pred[:, len(columns_class_name):]\n                regression_target = target[:, len(columns_class_name):]\n                regression_target[regression_target.isnan()] = regression_pred[regression_target.isnan()]\n                loss += self.tab_regression_loss(regression_pred, regression_target).mean()\n        else:\n            target[target.isnan()] = pred[target.isnan()]\n            loss += self.tab_regression_loss(pred, target).mean()\n        return loss\n\n    def embed(self, tab, img, txt) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:\n        \"\"\"Get the embeddings for each modality\"\"\"\n        if self.tab_embedder is None:\n            tab_embed = None\n        else:\n            tab = torch.tensor(tab.values).to(device=self.device, dtype=self.dtype)\n            tab_embed = self.tab_embedder(tab)\n        if self.img_embedder is None:\n            img_embed = None\n        else:\n            img = img.to(device=self.device, dtype=self.dtype)\n            img_embed = self.img_embedder(img)\n        if self.txt_embedder is None:\n            txt_embed = None\n        else:\n            self.txt_embedder.to(self.device)\n            txt_embed = self.txt_embedder(txt)\n        return (tab_embed, img_embed, txt_embed)\n\n    def decode(self, latent_embed: torch.Tensor) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:\n        \"\"\"Obtain logits / images / next tokens predictions given the latent embedding\"\"\"\n        if self.tab_head is None:\n            pred_tab = None\n        else:\n            pred_tab = self.tab_head(latent_embed)\n        if self.img_head is None:\n            pred_img = None\n        else:\n            pred_img = self.img_head(latent_embed)\n        if self.txt_head is None:\n            pred_txt = None\n        else:\n            pred_txt = self.txt_head(latent_embed)\n        return (pred_tab, pred_img, pred_txt)\n\n    def forward(self, tab: pd.DataFrame, img: torch.Tensor, txt: pd.DataFrame) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:\n        (tab_embed, img_embed, txt_embed) = self.embed(tab=tab, img=img, txt=txt)\n        to_fuse = [embed for embed in [tab_embed, img_embed, txt_embed] if embed is not None]\n        assert len(to_fuse) > 0\n        latent_embed = torch.cat(to_fuse, dim=1)\n        return self.decode(latent_embed=latent_embed)\n\n    def training_step(self, batch, batch_idx) -> torch.Tensor:\n        (indices, (tab_inputs_batch, img_inputs_batch, txt_inputs_batch), (tab_targets_batch, img_targets_batch, txt_targets_batch)) = batch\n        (pred_tab, pred_img, pred_txt) = self.forward(tab=tab_inputs_batch, img=img_inputs_batch, txt=txt_inputs_batch)\n        loss = 0.0\n        if tab_targets_batch is not None:\n            loss += self.tab_loss(pred=pred_tab, target=tab_targets_batch).mean()\n        if self.img_loss is not None:\n            loss += self.img_loss(pred_img, img_targets_batch).mean()\n        if self.txt_loss:\n            loss += self.txt_loss(pred_txt, txt_targets_batch).mean()\n        self.log('train_loss', loss, prog_bar=True, on_step=False, on_epoch=True, batch_size=len(indices))\n        if torch.isnan(loss).item():\n            raise NanLossError(f'Loss value is {loss}')\n        return loss\n\n    @staticmethod\n    def get_param_groups(model: nn.Module | None, **kwargs) -> dict[str, ...] | None:\n        if model is None:\n            return None\n        else:\n            return {'params': model.parameters(), **kwargs}\n\n    def configure_optimizers(self) -> torch.optim.Optimizer:\n        groups = [self.get_param_groups(model=self.tab_embedder, lr=self.learning_rate), self.get_param_groups(model=self.tab_head, lr=self.learning_rate), self.get_param_groups(model=self.img_embedder, lr=self.learning_rate), self.get_param_groups(model=self.img_head, lr=self.learning_rate), self.get_param_groups(model=self.txt_embedder, lr=self.learning_rate), self.get_param_groups(model=self.txt_head, lr=self.learning_rate)]\n        params = [group for group in groups if group is not None]\n        if self.optimizer_choice == 'adam':\n            optimizer = torch.optim.Adam(params, lr=self.learning_rate)\n        elif self.optimizer_choice == 'sgd':\n            optimizer = torch.optim.SGD(params, lr=self.learning_rate)\n        elif self.optimizer_choice == 'adamw':\n            optimizer = torch.optim.AdamW(params, lr=self.learning_rate)\n        else:\n            raise ValueError(f'Unsupported optimizer: {self.optimizer_choice}')\n        return optimizer\n\n    def on_train_start(self) -> None:\n        self.unfreeze_epoch = MAX_EPOCHS // 2\n\n    def on_train_epoch_start(self) -> None:\n        if self.current_epoch == self.unfreeze_epoch and self.img_embedder:\n            print(f'Unfreezing layers at epoch {self.current_epoch}')\n            self.img_embedder.unfreeze(n_last_layers=3)\n\n    def validation_step(self, batch, batch_idx):\n        \"\"\"Run\"\"\"\n        (indices, inputs_batch, targets_batch) = batch\n        (tab_inputs_batch, img_inputs_batch, txt_inputs_batch) = inputs_batch\n        (tab_targets_batch, img_targets_batch, txt_targets_batch) = targets_batch\n        preds_batch = self.forward(tab=tab_inputs_batch, img=img_inputs_batch, txt=txt_inputs_batch)\n        (tab_preds_batch, img_preds_batch, txt_preds_batch) = preds_batch\n        loss = 0.0\n        if self.tab_loss is not None:\n            loss += self.tab_loss(tab_preds_batch, tab_targets_batch)\n        if self.img_loss is not None:\n            loss += self.img_loss(img_preds_batch, img_targets_batch).mean()\n        if self.txt_loss:\n            loss += self.txt_loss(txt_preds_batch, txt_targets_batch).mean()\n        self.log('valid_loss', loss, prog_bar=True, on_step=False, on_epoch=True, batch_size=len(indices))\n\n    def get_submissions(self, dataloader: DataLoader, get_raw_preds: bool=False) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame | None]:\n        \"\"\" Get submissions for each element of the dataloader\n\n        Args:\n            dataloader: dataloader on which\n            get_raw_preds: whether to also return raw predictions\n        \"\"\"\n        submission = pd.DataFrame([])\n        raw_tab_preds = []\n        target = pd.DataFrame([])\n        img_preds = []\n        txt_preds = []\n        indices = []\n        tab_targets_batch = None\n        for batch in tqdm(dataloader):\n            (indices_batch, (tab_inputs_batch, img_inputs_batch, txt_inputs_batch), (tab_targets_batch, img_targets_batch, txt_targets_batch)) = batch\n            indices += list(indices_batch)\n            with torch.no_grad():\n                (pred_tab, pred_img, pred_txt) = self.forward(tab=tab_inputs_batch, img=img_inputs_batch, txt=txt_inputs_batch)\n            if pred_tab is not None:\n                if enc:\n                    softmax = nn.Softmax(dim=1)\n                    columns_class_name = list(enc.get_feature_names_out(class_names_columns_classification))\n                    classification_pred = pred_tab[:, :len(columns_class_name)].clone()\n                    groups_label = enc.categories_\n                    idx_start = 0\n                    for group in groups_label:\n                        group_pred = classification_pred[:, idx_start:idx_start + len(group)]\n                        pred_tab[:, idx_start:idx_start + len(group)] = softmax(group_pred)\n                        idx_start += len(group)\n                pred_tab = pred_tab.cpu().numpy()\n                if get_raw_preds:\n                    raw_tab_preds.append(pred_tab)\n                pred_tab = dataset.tab_target_inverse_transform(pred_tab, indices_batch)\n                submission = pd.concat([submission, pred_tab])\n            if pred_img is not None:\n                assert not isinstance(dataset.img_target_inverse_transform, Identity), 'Predicted images must be transformed'\n                pred_img = dataset.img_target_inverse_transform(pred_img)\n                img_preds.append(pred_img)\n            if pred_txt is not None:\n                pred_txt = dataset.txt_target_inverse_transform(pred_txt)\n                txt_preds.append(pred_txt)\n            if tab_targets_batch is not None and len(tab_targets_batch) > 0:\n                tab_targets_batch = dataset.tab_target_inverse_transform(tab_targets_batch.values, indices_batch)\n                target = pd.concat([target, tab_targets_batch])\n            if img_targets_batch is not None and len(img_targets_batch) > 0:\n                img_targets_batch = dataset.img_target_inverse_transform(img_targets_batch.values, indices_batch)\n                target = pd.concat([target, img_targets_batch])\n            if txt_targets_batch is not None and len(txt_targets_batch.columns) > 0:\n                txt_targets_batch = dataset.txt_target_inverse_transform(txt_targets_batch.values, indices_batch)\n                target = pd.concat([target, txt_targets_batch])\n        if dataset.custom_tab_regression_scaler is not None:\n            submission[dataset.tab_regression_target_cols] = dataset.custom_tab_regression_scaler.inverse_transform(submission[dataset.tab_regression_target_cols])\n            if tab_targets_batch is not None and len(tab_targets_batch) > 0:\n                target[dataset.tab_regression_target_cols] = dataset.custom_tab_regression_scaler.inverse_transform(target[dataset.tab_regression_target_cols])\n        if get_raw_preds:\n            raw_tab_preds = pd.DataFrame(np.concatenate(raw_tab_preds), index=indices)\n        else:\n            raw_tab_preds = None\n        return (submission, target, raw_tab_preds)\n\n    def get_blend_submissions(self, dataloader: DataLoader) -> tuple[list[Tensor], list[Tensor], list[...]]:\n        \"\"\"Get submissions for each element of the dataloader\"\"\"\n        embeddings_save = []\n        targets_save = []\n        indices = []\n        for batch in tqdm(dataloader):\n            (indices_batch, (tab_inputs_batch, img_inputs_batch, txt_inputs_batch), (tab_targets_batch, img_targets_batch, txt_targets_batch)) = batch\n            indices += list(indices_batch)\n            with torch.no_grad():\n                (tab_embed, img_embed, txt_embed) = self.embed(tab=tab_inputs_batch, img=img_inputs_batch, txt=txt_inputs_batch)\n                final_embedding = [embed for embed in [tab_embed, img_embed, txt_embed] if embed is not None]\n                assert len(final_embedding) > 0\n                latent_embed = torch.cat(final_embedding, dim=1)\n                embeddings_save.append(latent_embed)\n            batch_targets = []\n            if tab_targets_batch is not None and len(tab_targets_batch) > 0:\n                batch_targets.append(torch.tensor(tab_targets_batch.values, dtype=torch.float32))\n            if img_targets_batch is not None and len(img_targets_batch) > 0:\n                batch_targets.append(torch.tensor(img_targets_batch.values, dtype=torch.float32))\n            if txt_targets_batch is not None and len(txt_targets_batch.columns) > 0:\n                batch_targets.append(torch.tensor(txt_targets_batch.values, dtype=torch.float32))\n            if batch_targets:\n                targets_save.append(torch.cat(batch_targets, dim=1))\n        return (embeddings_save, targets_save, indices)\n\n    def get_score(self, dataloader: DataLoader) -> tuple[dict[str, float], pd.DataFrame]:\n        \"\"\" Iterate through the dataloader to get predictions in submission format and compute the score\n        Returns:\n            score\n            predictions in submission format\n        \"\"\"\n        (submissions, targets, _) = self.get_submissions(dataloader=dataloader)\n        scores = {}\n        for (submission_format_func, submission_name) in zip(submission_format_functions, submission_names):\n            try:\n                formatted_submission = submission_format_func(submissions)\n                formatted_targets = submission_format_func(targets)\n                try:\n                    score = metric_function(y_pred=formatted_submission, y_true=formatted_targets)\n                    score = float(score)\n                except (IndexError, TypeError):\n                    traceback.print_exc()\n                    score = np.nan\n                scores[submission_name] = score\n            except Exception as e:\n                print(f'Hit exception when trying to create {submission_name}: {e}')\n        return (scores, submissions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:52.559864Z","iopub.execute_input":"2025-03-11T15:56:52.560157Z","iopub.status.idle":"2025-03-11T15:56:52.592516Z","shell.execute_reply.started":"2025-03-11T15:56:52.560136Z","shell.execute_reply":"2025-03-11T15:56:52.591791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def is_main_process() -> bool:\n    \"\"\"\n    Check if the current process is the main process (rank 0).\n    \"\"\"\n    return not dist.is_initialized() or dist.get_rank() == 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:52.746691Z","iopub.execute_input":"2025-03-11T15:56:52.746968Z","iopub.status.idle":"2025-03-11T15:56:52.750851Z","shell.execute_reply.started":"2025-03-11T15:56:52.746948Z","shell.execute_reply":"2025-03-11T15:56:52.750131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(params: dict[str, ...], trial_num: int, accelerator: str, max_time: str, max_trials: int=10, devices: list[int] | str='auto') -> torch.Tensor:\n    learning_rate = params['learning_rate'][0]\n    optimizer_choice = params['optimizer'][0]\n    logger = TensorBoardLogger('tb_logs', name=f'hebo_run')\n    checkpoint_callback = ModelCheckpoint(save_top_k=1, monitor='valid_loss', mode='min', dirpath='./trials', filename=f'trial_{trial_num}')\n    early_stop_callback = EarlyStopping(monitor='valid_loss', min_delta=0.0, patience=5, verbose=True, mode='min')\n    train_batch_size = TRAIN_BATCH_SIZE\n    test_batch_size = TEST_BATCH_SIZE\n    (train_dataloader, validation_dataloader, _) = get_data_loaders(train_batch_size, test_batch_size)\n    val_loss = None\n    _trial = 0\n    while val_loss is None and _trial < max_trials:\n        model = SubmissionModel(learning_rate=learning_rate, optimizer_choice=optimizer_choice)\n        try:\n            extra_kwargs = dict(max_time=max_time, max_epochs=MAX_EPOCHS)\n            trainer = L.Trainer(accelerator=accelerator, devices=devices, logger=logger, callbacks=[checkpoint_callback, early_stop_callback], **extra_kwargs)\n            trainer.fit(model, train_dataloader, validation_dataloader)\n            val_loss = trainer.checkpoint_callback.best_model_score\n        except (RuntimeError, torch.cuda.OutOfMemoryError) as e:\n            if 'out of memory' in str(e).lower():\n                print('| WARNING: ran out of memory, retrying with half the batch size')\n                with torch.no_grad():\n                    torch.cuda.empty_cache()\n            train_batch_size //= 2\n            test_batch_size //= 2\n            if train_batch_size <= 0:\n                raise RuntimeError('No batch size seems to fit on the device')\n            (train_dataloader, validation_dataloader, _) = get_data_loaders(train_batch_size, test_batch_size)\n        finally:\n            _trial += 1\n    return val_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:53.083110Z","iopub.execute_input":"2025-03-11T15:56:53.083420Z","iopub.status.idle":"2025-03-11T15:56:53.090361Z","shell.execute_reply.started":"2025-03-11T15:56:53.083394Z","shell.execute_reply":"2025-03-11T15:56:53.089661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_trials(trials_dir: str) -> None:\n    if os.path.exists(trials_dir):\n        shutil.rmtree(trials_dir)\n        print(f'Deleted existing {trials_dir} folder.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:53.403695Z","iopub.execute_input":"2025-03-11T15:56:53.403916Z","iopub.status.idle":"2025-03-11T15:56:53.407576Z","shell.execute_reply.started":"2025-03-11T15:56:53.403897Z","shell.execute_reply":"2025-03-11T15:56:53.406904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_best_checkpoint(best_index: int, trials_dir: str='./trials') -> None:\n    best_model_path = os.path.join(trials_dir, f'trial_{best_index}.ckpt')\n    print('best_model_path', best_model_path)\n    if os.path.exists(best_model_path):\n        shutil.move(best_model_path, './best_model.ckpt')\n        print(f'Copied best model ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:53.771246Z","iopub.execute_input":"2025-03-11T15:56:53.771571Z","iopub.status.idle":"2025-03-11T15:56:53.775970Z","shell.execute_reply.started":"2025-03-11T15:56:53.771546Z","shell.execute_reply":"2025-03-11T15:56:53.775171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _format_submission_dtypes(formatted_submission: pd.DataFrame) -> pd.DataFrame:\n    sample_submission = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')\n    for column in sample_submission:\n        if sample_submission[column].isna().all():\n            try:\n                if np.array_equal(formatted_submission[column], formatted_submission[column].astype(int)):\n                    formatted_submission[column] = formatted_submission[column].round().astype(int)\n            except (ValueError, RuntimeError) as e:\n                print(e)\n                pass\n        else:\n            try:\n                if np.issubdtype(sample_submission[column].dtype, np.integer) and np.array_equal(formatted_submission[column], formatted_submission[column].astype(int)):\n                    formatted_submission[column] = formatted_submission[column].round().astype(int)\n            except (ValueError, RuntimeError) as e:\n                print(e)\n                pass\n            try:\n                if np.issubdtype(sample_submission[column].dtype, np.float32) and np.array_equal(formatted_submission[column], formatted_submission[column].astype(float)):\n                    formatted_submission[column] = formatted_submission[column].astype(float)\n            except (ValueError, RuntimeError) as e:\n                print(e)\n                pass\n    return formatted_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:54.123688Z","iopub.execute_input":"2025-03-11T15:56:54.123955Z","iopub.status.idle":"2025-03-11T15:56:54.129967Z","shell.execute_reply.started":"2025-03-11T15:56:54.123930Z","shell.execute_reply":"2025-03-11T15:56:54.129147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def time_to_seconds(time_str: str) -> float:\n    (days, hours, minutes, seconds) = map(int, time_str.split(':'))\n    total_seconds = days * 3600 * 24 + hours * 3600 + minutes * 60 + seconds\n    return total_seconds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:54.492124Z","iopub.execute_input":"2025-03-11T15:56:54.492391Z","iopub.status.idle":"2025-03-11T15:56:54.496054Z","shell.execute_reply.started":"2025-03-11T15:56:54.492368Z","shell.execute_reply":"2025-03-11T15:56:54.495263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seconds_to_time_string(total_seconds: float) -> str:\n    days = int(total_seconds // (3600 * 24))\n    hours = int(total_seconds // 3600)\n    minutes = int(total_seconds % 3600 // 60)\n    seconds = int(total_seconds % 60)\n    time_string = f'{days:02}:{hours:02}:{minutes:02}:{seconds:02}'\n    return time_string","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:54.825920Z","iopub.execute_input":"2025-03-11T15:56:54.826154Z","iopub.status.idle":"2025-03-11T15:56:54.830251Z","shell.execute_reply.started":"2025-03-11T15:56:54.826135Z","shell.execute_reply":"2025-03-11T15:56:54.829561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def graceful_exit(start_time: float, max_exec_time: float, time_file: str, run_out_time_file: str) -> None:\n    time_taken = time.time() - start_time\n    remaining_time = max_exec_time - time_taken\n    with open(time_file, 'w') as f:\n        f.write(str(remaining_time))\n    if remaining_time < 0:\n        print('Total execution crossed allocated time')\n        with open(run_out_time_file, 'w') as f:\n            f.write(str(remaining_time))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:55.179831Z","iopub.execute_input":"2025-03-11T15:56:55.180058Z","iopub.status.idle":"2025-03-11T15:56:55.184537Z","shell.execute_reply.started":"2025-03-11T15:56:55.180039Z","shell.execute_reply":"2025-03-11T15:56:55.183664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_max_time_limit(time_file: str, margin_time: int=900) -> tuple[float, str]:\n    \"\"\"\n    Find maximum time allowed to execute this script\n    :param margin_time: Margin time to accommodate test generation\n    :param time_file: File containing allowed execution time\n    :return: tuple containing maximum execution time and allowed training time\n    \"\"\"\n    max_training_time = MAX_TIME\n    max_exec_time = 2 * 24 * 3600\n    if os.path.exists(time_file):\n        try:\n            with open(time_file, 'r') as f:\n                time_str = f.read()\n                if len(time_str) > 0:\n                    max_exec_time = float(time_str) - margin_time\n                    assert max_exec_time > 0, f'Exhausted allowed execution time {max_exec_time}'\n        except Exception as e:\n            raise e\n    if time_to_seconds(MAX_TIME) > max_exec_time:\n        max_training_time = seconds_to_time_string(max_exec_time)\n    return (max_exec_time, max_training_time)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:55.549125Z","iopub.execute_input":"2025-03-11T15:56:55.549524Z","iopub.status.idle":"2025-03-11T15:56:55.554704Z","shell.execute_reply.started":"2025-03-11T15:56:55.549491Z","shell.execute_reply":"2025-03-11T15:56:55.553846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main(accelerator: str, devices: str) -> None:\n    if os.getenv('RUN_TTA_ONLY', False) in ['1', 'true', 'True'] or os.getenv('RUN_INFERENCE_ONLY', False) in ['1', 'true', 'True']:\n        return generate_submissions()\n    start_time: float = time.time()\n    time_file = Path('./remaining_time.txt')\n    run_out_time_file = time_file.parent / 'run_out_of_time.txt'\n    count_free_gpus = [device_id for device_id in range(torch.cuda.device_count()) if torch.cuda.utilization(device_id) == 0]\n    accelerator = accelerator if len(count_free_gpus) else 'cpu'\n    if os.path.exists(time_file):\n        (max_exec_time, max_training_time) = get_max_time_limit(time_file=str(time_file))\n        print(f'Maximum allowed execution time : {max_exec_time} seconds and training time {max_training_time}')\n        atexit.register(partial(graceful_exit, start_time=start_time, max_exec_time=max_exec_time, time_file=time_file, run_out_time_file=run_out_time_file))\n    else:\n        max_exec_time = 2 * 24 * 3600\n        max_training_time = MAX_TIME\n    fast_model = SubmissionModel()\n    fast_trainer = L.Trainer(accelerator=accelerator, devices=devices, fast_dev_run=True)\n    fast_train_batch_size = TRAIN_BATCH_SIZE\n    fast_test_batch_size = TEST_BATCH_SIZE\n    (fast_train_dataloader, fast_validation_dataloader, _) = get_data_loaders(train_batch_size=fast_train_batch_size, val_batch_size=fast_test_batch_size)\n    fast_train_val_loss = None\n    _fast_trials = 0\n    _max_fast_trials = 5\n    print(f'BEFORE FAST TRAINING: max_exec_time: {max_exec_time}, max_training_time: {max_training_time}', flush=True)\n    while fast_train_val_loss is None and _fast_trials < _max_fast_trials:\n        try:\n            print('STARTING FAST TRAINING', flush=True)\n            _fast_trials += 1\n            fast_trainer.fit(fast_model, fast_train_dataloader, fast_validation_dataloader)\n            fast_train_val_loss = fast_trainer.callback_metrics['valid_loss'].item()\n        except (RuntimeError, torch.cuda.OutOfMemoryError) as e:\n            if 'out of memory' in str(e).lower():\n                print('| WARNING: ran out of memory, retrying with half the batch size', flush=True)\n                with torch.no_grad():\n                    torch.cuda.empty_cache()\n            elapsed_time = time.time() - start_time\n            if elapsed_time > max_exec_time:\n                print('Total time exceeded. Stopping trials.', flush=True)\n                break\n            fast_train_batch_size //= 2\n            fast_test_batch_size //= 2\n            if fast_train_batch_size <= 0:\n                raise RuntimeError('No batch size seems to make it fit on the device')\n            (fast_train_dataloader, fast_validation_dataloader, _) = get_data_loaders(fast_train_batch_size, fast_test_batch_size)\n    elapsed_time = time.time() - start_time\n    max_exec_time = max_exec_time - elapsed_time\n    if time_to_seconds(max_training_time) > max_exec_time - elapsed_time:\n        max_training_time = seconds_to_time_string(max_exec_time - elapsed_time)\n    print(f'AFTER FAST TRAINING - max_exec_time: {max_exec_time}, max_training_time: {max_training_time}, elapse_time: {elapsed_time}', flush=True)\n    start_time = time.time()\n    if N_TRIALS == 0:\n        trials_dir = './trials'\n        best_index = 0\n        params = {'learning_rate': [0.0001], 'optimizer': ['adam']}\n        val_loss = objective(params=params, trial_num=0, accelerator=accelerator, max_time=max_training_time)\n    else:\n        trials_dir = './trials'\n        predefined_suggestions = [{'learning_rate': [0.0001], 'optimizer': ['adam']}, {'learning_rate': [0.0001], 'optimizer': ['sgd']}, {'learning_rate': [0.0001], 'optimizer': ['adamw']}]\n        opt = HEBO(HEBO_SPACE)\n        remove_trials(trials_dir)\n        for trial in range(N_TRIALS):\n            elapsed_time = time.time() - start_time\n            if elapsed_time > max_exec_time:\n                print('Total time exceeded. Stopping trials.')\n                break\n            if time_to_seconds(max_training_time) > max_exec_time - elapsed_time:\n                max_training_time = seconds_to_time_string(max_exec_time - elapsed_time)\n            if is_main_process():\n                if len(predefined_suggestions) > 0:\n                    params = predefined_suggestions.pop(0)\n                    rec = pd.DataFrame.from_dict(params, orient='columns')\n                else:\n                    rec = opt.suggest(n_suggestions=1)\n                    params = rec.to_dict(orient='list')\n            try:\n                val_loss = objective(params=params, trial_num=trial, accelerator=accelerator, max_time=max_training_time, devices=devices)\n                if dist.is_initialized():\n                    raise NotImplementedError(f'HEBO with DDP training not supported yet - TODO: synchronize HEBO kernel params across ranks')\n                    val_loss = val_loss.reshape((1,)).to(f'cuda:{dist.get_rank()}')\n                    val_losses = [torch.tensor([0.0], device=f'cuda:{dist.get_rank()}') for _ in range(dist.get_world_size())]\n                    dist.all_gather(val_losses, val_loss)\n                    val_loss_gathered = torch.cat(val_losses).mean().cpu().detach()\n                    opt.observe(rec, np.array([[val_loss_gathered.item()]]))\n                else:\n                    opt.observe(rec, np.array([[val_loss.item()]]))\n            except NanLossError as e:\n                print(f'Trial {trial} failed: {e}, stopping the trials.')\n                break\n            except Exception as e:\n                print(f'Trial {trial} failed: {e}')\n            opt.X.to_csv('./hyperopt_x.csv')\n            pd.DataFrame(opt.y).to_csv('./hyperopt_y.csv')\n        if opt.X.shape[0] >= 1:\n            best_params = opt.X.iloc[opt.y.argmin()].to_dict()\n            best_index = opt.y.argmin()\n            print(f'Best hyper parameters: {best_params}')\n            print(f'Best index: {best_index}')\n    if is_main_process():\n        get_best_checkpoint(best_index=best_index, trials_dir=trials_dir)\n        generate_submissions()\n        remove_trials(trial_directory)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:56:55.900765Z","iopub.execute_input":"2025-03-11T15:56:55.901039Z","iopub.status.idle":"2025-03-11T15:56:55.915398Z","shell.execute_reply.started":"2025-03-11T15:56:55.901018Z","shell.execute_reply":"2025-03-11T15:56:55.914635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_submissions() -> None:\n    model = SubmissionModel.load_from_checkpoint(checkpoint_path=\"/kaggle/input/solution_model/pytorch/default/1/best_model.ckpt\")\n    model.eval()\n    free_gpus = [device_id for device_id in range(torch.cuda.device_count()) if torch.cuda.utilization(device_id) == 0]\n    print(f'Free GPUs: {free_gpus}')\n    model.to(free_gpus[0] if len(free_gpus) else 'cpu')\n    run_tta_only = os.getenv('RUN_TTA_ONLY', False) in ['1', 'true', 'True']\n    current_dir = Path().resolve()\n    if not run_tta_only:\n        (val_scores, validation_submission) = model.get_score(validation_dataloader)\n        save_path = str(current_dir / 'val_scores.json')\n        with open(save_path, 'w') as writer:\n            writer.write(json.dumps(val_scores))\n    print(f'[START] Generate test predictions')\n    use_tta = os.getenv('TTA', False) in ['1', 'true', 'True'] or run_tta_only\n    (test_submission, _, no_tta_raw_preds) = model.get_submissions(dataloader=test_dataloader, get_raw_preds=use_tta)\n    n_submissions = 0\n    for (submission_format_func, submission_name) in zip(submission_format_functions, submission_names):\n        try:\n            formatted_sub = submission_format_func(test_submission)\n            formatted_sub = _format_submission_dtypes(formatted_sub)\n            formatted_sub.to_csv(os.path.join(str(current_dir), submission_name), index=False)\n            n_submissions += 1\n        except Exception as e:\n            print(f'Hit exception when trying to create {submission_name}: {e}')\n    if use_tta:\n        print(f'[START] Generate test predictions using TTA')\n        tta_dir = current_dir / 'tta'\n        os.makedirs(tta_dir, exist_ok=True)\n        predictions = {'test_transform': no_tta_raw_preds}\n        from solve_params import TTA_ROUNDS\n        test_dataset.img_input_transform = CustomTrainImageInputTransform\n        for tta_round in tqdm(range(TTA_ROUNDS), desc='Generating TTA predictions'):\n            (test_submission, _, raw_preds) = model.get_submissions(dataloader=test_dataloader, get_raw_preds=True)\n            for (submission_format_func, submission_name) in zip(submission_format_functions, submission_names):\n                try:\n                    formatted_sub = submission_format_func(test_submission)\n                    formatted_sub = _format_submission_dtypes(formatted_sub)\n                    submission_name = f'tta_round_{tta_round}-' + submission_name\n                    predictions[submission_name] = raw_preds.loc[no_tta_raw_preds.index]\n                    formatted_sub.to_csv(tta_dir / submission_name, index=False)\n                except Exception as e:\n                    print(f'Hit exception when trying to create {submission_name}: {e}')\n        raw_preds_mean = np.stack(list(predictions.values())).mean(0)\n        tta_submission_transform = tab_target_inverse_transform(raw_preds_mean, no_tta_raw_preds.index.values)\n        if test_dataset.custom_tab_regression_scaler is not None:\n            tta_submission_transform[test_dataset.tab_regression_target_cols] = test_dataset.custom_tab_regression_scaler.inverse_transform(tta_submission_transform[test_dataset.tab_regression_target_cols])\n        for (submission_format_func, submission_name) in zip(submission_format_functions, submission_names):\n            try:\n                formatted_sub = submission_format_func(tta_submission_transform)\n                formatted_sub = _format_submission_dtypes(formatted_sub)\n                submission_name = f'tta-' + submission_name\n                formatted_sub.to_csv(current_dir / submission_name, index=False)\n            except Exception as e:\n                print(f'Hit exception when trying to create {submission_name}: {e}')\n    if n_submissions > 0:\n        print(f'[END] Generate test predictions in {current_dir}')\n    else:\n        print(f'[END] Failed to create a submission in {current_dir}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:57:02.524783Z","iopub.execute_input":"2025-03-11T15:57:02.525172Z","iopub.status.idle":"2025-03-11T15:57:02.541611Z","shell.execute_reply.started":"2025-03-11T15:57:02.525139Z","shell.execute_reply":"2025-03-11T15:57:02.540662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\naccelerator = 'gpu'\ndevices = 'auto'\nos.environ['RUN_INFERENCE_ONLY'] = 'True'\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"1\"\nif os.getenv('AGENT_DEBUG', False) in ['True', 'true', '1']:\n    MAX_EPOCHS = 1\n    N_TRIALS = 0\n    MAX_TIME = '00:00:01:00'\n    (train_dataloader, validation_dataloader, test_dataloader) = get_data_loaders(TRAIN_BATCH_SIZE, TEST_BATCH_SIZE)\nelif str(os.getenv('AGENT_NO_BO', False)) in ['True', 'true', '1']:\n    N_TRIALS = 0\n    MAX_EPOCHS = 30\n    MAX_TIME = '00:10:00:00'\nelse:\n    MAX_EPOCHS = 30\n    MAX_TIME = '00:10:00:00'\nmain(accelerator=accelerator, devices=devices)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T15:57:02.906771Z","iopub.execute_input":"2025-03-11T15:57:02.907155Z","iopub.status.idle":"2025-03-11T15:57:44.019842Z","shell.execute_reply.started":"2025-03-11T15:57:02.907122Z","shell.execute_reply":"2025-03-11T15:57:44.018870Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}