{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 📦 Install Required Packages\n# In this cell, we're installing essential Python packages using pip.\n# We are adding 'torch-geometric' and 'torch-scatter' packages to the environment.\n# These packages are crucial for graph-based machine learning tasks.\n\n!pip install torch-geometric torch-scatter\n","metadata":{"_kg_hide-output":true,"papermill":{"duration":264.245878,"end_time":"2023-09-01T16:55:11.276628","exception":false,"start_time":"2023-09-01T16:50:47.03075","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T08:25:42.887318Z","iopub.execute_input":"2023-09-13T08:25:42.887604Z","iopub.status.idle":"2023-09-13T08:37:40.029807Z","shell.execute_reply.started":"2023-09-13T08:25:42.887578Z","shell.execute_reply":"2023-09-13T08:37:40.028453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 📚 Import Necessary Libraries\n# In this cell, we import essential Python libraries to support various tasks in our notebook.\n# We use libraries like NumPy, Pandas, tqdm, scikit-learn, PyTorch, and others.\n# The 'device' variable is set to 'cuda' if a GPU is available; otherwise, it defaults to 'cpu'.\n\nimport numpy as np  # 🧮 NumPy for numerical computations\nimport pandas as pd  # 🐼 Pandas for data manipulation\nimport os  # 📂 Operating system-related functions\nfrom tqdm import tqdm  # 🔄 tqdm for progress bar visualization\n\nimport sklearn  # 🧬 scikit-learn for machine learning utilities\nimport sklearn.model_selection  # 📊 scikit-learn's model selection module\nimport torch  # 🔥 PyTorch for deep learning\nfrom torch import nn  # 🧠 PyTorch's neural network module\nfrom torch import Tensor  # 🚀 PyTorch's Tensor data type\nfrom torch_geometric.nn import GCNConv  # 📊 Graph Convolutional Network layer\nfrom torch_geometric.datasets import Planetoid  # 🌍 PyTorch Geometric dataset for graph data\nfrom torch.utils.data import DataLoader, Dataset  # 📦 PyTorch data loading utilities\nfrom timm.scheduler import CosineLRScheduler  # 📈 Learning rate scheduler\nimport matplotlib.pyplot as plt  # 📊 Matplotlib for plotting\nimport torch.nn.functional as F\nimport math\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'  # ⚙️ Determine if CUDA (GPU) is available\n","metadata":{"papermill":{"duration":4.819384,"end_time":"2023-09-01T16:55:16.104784","exception":false,"start_time":"2023-09-01T16:55:11.2854","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:25.208234Z","iopub.execute_input":"2023-09-13T09:35:25.208978Z","iopub.status.idle":"2023-09-13T09:35:25.217358Z","shell.execute_reply.started":"2023-09-13T09:35:25.208941Z","shell.execute_reply":"2023-09-13T09:35:25.216370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 📁 Define a Function to Load DataFrames\n# This function loads data stored in different splits (train, valid, test) from a specified directory.\n# It reads files in the directory, extracts data using NumPy, and organizes it into DataFrames.\n\ndef load_df(directory):\n    splits = [\"train\", \"valid\", \"test\"]  # 🔄 List of data splits\n    dfs = dict()  # 📊 Dictionary to store DataFrames for each split\n    \n    for split in splits:\n        path = os.path.join(directory, split)  # 📂 Define the path to the split's directory\n        files = os.listdir(path)  # 🗂 Get a list of files in the split's directory\n        list_df = []  # 📄 List to store data dictionaries\n        \n        for file in files:\n            d = dict(np.load(os.path.join(path, file)))  # 📦 Load data using NumPy\n            d['file'] = file  # 📄 Include the file name in the data dictionary\n            list_df.append(d)  # 🧾 Append the data dictionary to the list\n        dfs[split] = pd.DataFrame.from_dict(list_df)  # 🐼 Create a DataFrame from the list of data dictionaries and store it in the dictionary\n    return dfs\n\n# 📄 Load data using the defined function and store it in the 'tile_xla' variable\ntile_xla = load_df(\"/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/\")\n","metadata":{"papermill":{"duration":0.020227,"end_time":"2023-09-01T16:55:16.152594","exception":false,"start_time":"2023-09-01T16:55:16.132367","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:25.219114Z","iopub.execute_input":"2023-09-13T09:35:25.219734Z","iopub.status.idle":"2023-09-13T09:35:57.384417Z","shell.execute_reply.started":"2023-09-13T09:35:25.219651Z","shell.execute_reply":"2023-09-13T09:35:57.383386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📦 Define Dataset and Model","metadata":{"papermill":{"duration":0.008874,"end_time":"2023-09-01T16:56:21.968592","exception":false,"start_time":"2023-09-01T16:56:21.959718","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 📦 Define Custom Dataset Class\n# This class, 'TileDataset', is a custom dataset class for our machine learning task.\n# It inherits from the PyTorch 'Dataset' class and implements the necessary methods (__init__, __len__, and __getitem__).\n\nclass TileDataset(Dataset):\n    def __init__(self, df):\n        self.df = df  # 💼 Initialize the dataset with a DataFrame containing the data\n\n    def __len__(self):\n        return len(self.df)  # 🔢 Define the length of the dataset, which is the number of rows in the DataFrame\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]  # 📄 Get a specific row from the DataFrame based on the provided index\n        config_feat = torch.tensor(row['config_feat'].astype(np.float32))  # 🧮 Convert and store 'config_feat' as a PyTorch tensor\n        node_feat = torch.tensor(row['node_feat'].astype(np.float32))  # 🧮 Convert and store 'node_feat' as a PyTorch tensor\n        node_opcode = torch.tensor(row['node_opcode'].astype(np.int32))  # 🧮 Convert and store 'node_opcode' as a PyTorch tensor\n        edge_index = torch.tensor(np.swapaxes(row['edge_index'],0,1).astype(np.int32))  # 🧮 Convert and store 'edge_index' as a PyTorch tensor with axis swapping\n        target = (row['config_runtime'] / (row['config_runtime_normalizers'] + 1e-5)).astype(np.float32)  # 📈 Calculate and store the target value with preprocessing\n        # 📊 Min-max scale the target value to ensure it's within a specific range (standardization)\n        target = (target - np.mean(target)) / (np.std(target) + 1e-5)\n        target = torch.tensor(target)  # 🧮 Convert and store the target as a PyTorch tensor\n        return config_feat, node_feat, node_opcode, edge_index, target  # 🔁 Return the data and target for a specific sample\n\n# This class defines the structure of our custom dataset, converting and preprocessing data as necessary for training and evaluation.\n# The relevant emojis provide a visual context for each part of the code.\n","metadata":{"papermill":{"duration":0.020329,"end_time":"2023-09-01T16:56:21.997734","exception":false,"start_time":"2023-09-01T16:56:21.977405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:57.386421Z","iopub.execute_input":"2023-09-13T09:35:57.386780Z","iopub.status.idle":"2023-09-13T09:35:57.398847Z","shell.execute_reply.started":"2023-09-13T09:35:57.386746Z","shell.execute_reply":"2023-09-13T09:35:57.397835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_seq_len):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=0.1)\n        \n        pe = torch.zeros(max_seq_len, d_model)\n        position = torch.arange(0, max_seq_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n#         print('pe',pe.shape)\n#         pe = pe.unsqueeze(0).transpose(0, 1)\n#         print('pe',pe.shape)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        x = x + self.pe[:x.size(0), :]\n        return self.dropout(x)\n\nclass TransformerClassifier(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim, num_heads=8, num_layers=6):\n        super(TransformerClassifier, self).__init__()\n        self.embedding = nn.Embedding(input_dim, hidden_dim)\n        self.embedding2 = nn.Linear(input_dim, hidden_dim)\n        self.position_encoding = PositionalEncoding(hidden_dim, max_seq_len=500) # 100\n        encoder_layer = nn.TransformerEncoderLayer(hidden_dim, num_heads)\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    def forward(self, x):\n#         print('1',x.shape)\n#         x = self.embedding(x.long())\n        x = self.embedding2(x)\n#         print('2',x.shape)\n        x = self.position_encoding(x)\n#         print('3',x.shape)\n#         x = x.permute(1, 0, 2)\n        x = self.transformer_encoder(x)\n#         print('4',x.shape)\n#         x = x.mean(dim=1)  # Average the sequence representations\n        x = self.fc(x).sigmoid()\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-09-13T09:35:57.400963Z","iopub.execute_input":"2023-09-13T09:35:57.401739Z","iopub.status.idle":"2023-09-13T09:35:57.418805Z","shell.execute_reply.started":"2023-09-13T09:35:57.401706Z","shell.execute_reply":"2023-09-13T09:35:57.417645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 🧠 Define Simple Neural Network Model\n# In this cell, we define a simple neural network model named 'SimpleModel'.\n# This model takes input data with specified dimensions and passes it through convolutional and dense layers.\n\nclass SimpleModel(torch.nn.Module):\n    def __init__(self, hidden_channels, graph_feats, hidden_dim):\n        super().__init__()  # 🧬 Initialize the parent class 'torch.nn.Module'\n        \n        op_embedding_dim = 4  # I choose 4-dimensional embedding\n        self.embedding = torch.nn.Embedding(120,  # 120 different op-codes\n                                            op_embedding_dim,\n                                           )\n        assert len(hidden_channels) > 0\n        in_channels = op_embedding_dim + 140\n        self.convs = torch.nn.ModuleList()\n        last_dim = hidden_channels[0]\n        \n        # Create a sequence of Graph Convolutional Network (GCN) layers\n        self.convs.append(GCNConv(in_channels, hidden_channels[0]))\n        for i in range(len(hidden_channels) - 1):\n            self.convs.append(GCNConv(hidden_channels[i], hidden_channels[i+1]))\n            last_dim = hidden_channels[i+1]\n        self.convs.append(GCNConv(last_dim, graph_feats))\n        \n        # Define a sequential dense neural network\n        self.dense = torch.nn.Sequential(nn.Linear(graph_feats + 24, 64),\n                                         nn.ReLU(),\n                                         nn.Linear(64, 64),\n                                         nn.ReLU(),\n#                                          nn.Linear(64, 16),\n#                                          nn.ReLU(),\n#                                          nn.Linear(16, 64),\n                                         nn.ReLU(),\n                                         nn.Linear(64, 1),\n                                        )\n        self.new_conv = GCNConv(in_channels, 64)\n        self.transformer_module = TransformerClassifier(64,64,64)\n\n    def forward(self, x_cfg: Tensor, x_feat: Tensor, x_op: Tensor, edge_index: Tensor) -> Tensor:\n        \n        # Get graph features\n        x = torch.cat([x_feat, self.embedding(x_op)], dim=1)  # 📊 Concatenate input features with opcode embeddings\n        \n#         print('x.shape: ', x.shape)\n        \n        # Pass data through convolutional layers\n        for i, conv in enumerate(self.convs):\n            x = conv(x, edge_index).relu()\n#             if i == 0 or i >= len(self.convs)-1:\n#                 x = conv(x, edge_index).relu()\n# #                 print(x.shape)\n\n#             else:\n#                 last_data = x\n#                 x = conv(x, edge_index).relu()\n#                 x = x + last_data\n\n#             elif i % 2 == 1:\n#                 last_data = x\n#                 x = conv(x, edge_index).relu()\n# #                 print(x.shape)\n#             else:\n#                 x = conv(x, edge_index).relu()\n# #                 print(x.shape)\n# #                 x += last_data\n#                 x = x + last_data\n\n#         x = self.new_conv(x, edge_index).relu()\n#         print(x)\n#         x = self.transformer_module(x)\n#         print(x)\n            \n#         print('after x.shape', x.shape)\n        \n        # Get 1D graph embedding using average pooling\n        x_graph = torch.mean(x, 0)\n         \n#         print('mean x.shape', x.shape)\n        \n        # Combine graph data with config data\n        x = torch.cat([x_cfg, x_graph.repeat((len(x_cfg), 1))], axis=1)  # 🔄 Concatenate config data with repeated graph embeddings\n        \n        # Pass the combined data through the dense neural network\n        x = torch.flatten(self.dense(x))\n        \n        # Standardize the output\n        x = (x - torch.mean(x)) / (torch.std(x) + 1e-5)\n        return x\nprint(device)\n# Create an instance of the 'SimpleModel' and move it to the specified device (CPU or GPU)\nmodel = SimpleModel(hidden_channels=[4,4,4], graph_feats=64, hidden_dim=64).to(device)\n","metadata":{"papermill":{"duration":0.063336,"end_time":"2023-09-01T16:56:22.069894","exception":false,"start_time":"2023-09-01T16:56:22.006558","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T11:37:56.384888Z","iopub.execute_input":"2023-09-13T11:37:56.385318Z","iopub.status.idle":"2023-09-13T11:37:56.430809Z","shell.execute_reply.started":"2023-09-13T11:37:56.385282Z","shell.execute_reply":"2023-09-13T11:37:56.429807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚂 Train One Epoch","metadata":{"papermill":{"duration":0.008439,"end_time":"2023-09-01T16:56:22.088164","exception":false,"start_time":"2023-09-01T16:56:22.079725","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 📊 Concatenate DataFrames\n# In this cell, we concatenate DataFrames 'train' and 'valid' from the 'tile_xla' dictionary along the row axis.\n# We then reset the index of the resulting DataFrame for consistent indexing.\n\n# Concatenate 'train' and 'valid' DataFrames along the row axis and reset the index\ndf = pd.concat((tile_xla[\"train\"], tile_xla[\"valid\"]), axis=0).reset_index(drop=True)\n\n# This operation combines the training and validation data for further processing, ensuring a unified DataFrame.\n","metadata":{"papermill":{"duration":0.021726,"end_time":"2023-09-01T16:56:22.11899","exception":false,"start_time":"2023-09-01T16:56:22.097264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:58.981744Z","iopub.execute_input":"2023-09-13T09:35:58.982114Z","iopub.status.idle":"2023-09-13T09:35:58.991154Z","shell.execute_reply.started":"2023-09-13T09:35:58.982078Z","shell.execute_reply":"2023-09-13T09:35:58.990083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 🔄 Cross-Validation Training Loop\n# In this cell, we implement a cross-validation training loop to train and evaluate the model on different data splits.\n\n# Create a K-Fold cross-validator with 5 splits\nkfold = sklearn.model_selection.KFold(n_splits=6, shuffle=True, random_state=0)\n\n# Lists to store mean and max scores for each fold\nscore_means = []\nscore_maxs = []\n\nbs = 1\n\n# Iterate through each fold\nfor fold, (tr_idx, va_idx) in enumerate(kfold.split(df)):\n    if fold != 0:\n        break\n    train_dataset = TileDataset(df.iloc[tr_idx])  # 📊 Create a training dataset\n    val_dataset = TileDataset(df.iloc[va_idx])  # 📊 Create a validation dataset\n#     criterion = torch.nn.MSELoss()  # 📉 Define the Mean Squared Error loss function\n    criterion = torch.nn.HuberLoss()\n    steps = len(train_dataset) / bs\n#     steps = len(train_dataset)\n    warmup_steps = int(steps * 0.3)\n    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4, weight_decay=1e-5)  # 🛠️ Define the optimizer\n    scheduler = CosineLRScheduler(optimizer, t_initial=steps, warmup_t=warmup_steps, warmup_lr_init=1e-6, lr_min=2e-8)  # 📈 Define the learning rate scheduler\n\n    # Define a function to calculate the mean score for tiles\n    def score_tile_mean(predictions, df):\n        score = 0\n        for i in range(len(df)):\n            predbest = np.mean(df.iloc[i]['config_runtime'][predictions[i]])\n            best = np.mean(np.sort(df.iloc[i]['config_runtime'])[:5])\n            score += 2 - predbest / best\n        score /= len(df)\n        return score\n\n    # Define a function to calculate the max score for tiles\n    def score_tile_max(predictions, df):\n        score = 0\n        for i in range(len(df)):\n            predbest = np.min(df.iloc[i]['config_runtime'][predictions[i]])\n            best = np.min(df.iloc[i]['config_runtime'])\n            score += 2 - predbest / best\n        score /= len(df)\n        return score\n\n    best_score = 0\n    best_score_max = 0\n\n    # Training loop\n    for epoch in range(10):\n        model.train()\n        pbar = tqdm(range(len(train_dataset)//bs), leave=False)\n        loss_sum = 0\n        n = 0\n        \n        for i in pbar:\n            if (i+1)*bs < len(train_dataset):\n                get = False\n                for j in range(bs):\n                    if not get:\n                        cfg_ft, nd_ft, nd_op, ind, target = train_dataset[i*bs+j]\n                        get = True\n#                         print('aaaa',ind.shape)\n                    else:\n                        cfg_ft1, nd_ft1, nd_op1, ind1, target1 = train_dataset[i*bs+j]\n#                         print('bbbb',ind1.shape)\n                        cfg_ft = torch.cat((cfg_ft, cfg_ft1), dim=0)\n                        nd_ft = torch.cat((nd_ft, nd_ft1), dim=0)\n                        nd_op = torch.cat((nd_op, nd_op1), dim=0)\n                        ind = torch.cat((ind, ind1), dim=-1)\n#                         print('cccc',ind.shape)\n                        target = torch.cat((target, target1), dim=0)\n            else:\n                get = False\n                for j in range(bs):\n                    if not get:\n                        cfg_ft, nd_ft, nd_op, ind, target = train_dataset[(i-1)*bs+j]\n                        get = True\n                    else:\n                        cfg_ft1, nd_ft1, nd_op1, ind1, target1 = train_dataset[(i-1)*bs+j]\n                        cfg_ft = torch.cat((cfg_ft, cfg_ft1), dim=0)\n                        nd_ft = torch.cat((nd_ft, nd_ft1), dim=0)\n                        nd_op = torch.cat((nd_op, nd_op1), dim=0)\n                        ind = torch.cat((ind, ind1), dim=-1)\n                        target = torch.cat((target, target1), dim=0)\n            \n            cfg_ft, nd_ft, nd_op, ind, target = cfg_ft.to(device), nd_ft.to(device), nd_op.to(device), ind.to(device), target.to(device)\n            \n            out = model(cfg_ft, nd_ft, nd_op, ind)\n            loss = criterion(out, target)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 2e-1)\n            scheduler.step(i + len(train_dataset) * epoch)\n            optimizer.step()\n            loss_sum += loss.item()\n            n += 1\n            pbar.set_description(f'running loss: {(loss_sum/n):.4f}, current loss: {(loss.item()):.4f}')\n        pbar.close()\n        model.eval()\n        tile_xla_predictions = []\n        pbar = tqdm(range(len(val_dataset)), leave=False)\n        \n        for i in pbar:\n            cfg_ft, nd_ft, nd_op, ind, target = val_dataset[i]\n            cfg_ft, nd_ft, nd_op, ind, target = cfg_ft.to(device), nd_ft.to(device), nd_op.to(device), ind.to(device), target.to(device)\n            \n            out = model(cfg_ft, nd_ft, nd_op, ind)\n            tile_xla_predictions.append(np.argsort(out.cpu().detach().numpy())[:5])\n        pbar.close()\n        \n        # Calculate and display scores for the current fold and epoch\n        score_mean = score_tile_mean(tile_xla_predictions, val_dataset.df)\n        score_max = score_tile_max(tile_xla_predictions, val_dataset.df)\n        print(f'fold {fold} epoch {epoch}, comp_score = {score_max:.3f}, mean_score = {score_mean:.3f},')\n        \n        # Update best scores and save the model if the mean score improves\n#         if score_mean > best_score:\n        if score_max > best_score_max:\n            best_score = score_mean\n            best_score_max = score_max\n            torch.save(model.state_dict(), f'best_model_{fold}.pth')\n    \n    # Append the best scores for this fold to the respective lists\n    score_means.append(best_score)\n    score_maxs.append(best_score_max)\n\n# Calculate and display the mean scores across all folds\nprint(f'comp_score = {np.mean(score_maxs)}, mean_score = {np.mean(score_means)},')\n","metadata":{"papermill":{"duration":9138.052878,"end_time":"2023-09-01T19:28:40.180685","exception":false,"start_time":"2023-09-01T16:56:22.127807","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T11:42:07.320974Z","iopub.execute_input":"2023-09-13T11:42:07.321436Z","iopub.status.idle":"2023-09-13T11:48:50.143311Z","shell.execute_reply.started":"2023-09-13T11:42:07.321402Z","shell.execute_reply":"2023-09-13T11:48:50.141622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":9138.052878,"end_time":"2023-09-01T19:28:40.180685","exception":false,"start_time":"2023-09-01T16:56:22.127807","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📊 Evaluate on Validation Dataset","metadata":{"papermill":{"duration":20.903162,"end_time":"2023-09-01T19:29:21.465429","exception":false,"start_time":"2023-09-01T19:29:00.562267","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 🚀 Predict and Submit (only tile:xla predictions)","metadata":{"papermill":{"duration":20.552961,"end_time":"2023-09-01T19:30:43.664106","exception":false,"start_time":"2023-09-01T19:30:23.111145","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 📊 Predict on Test Dataset (tile:xla)\n# In this section, we use the trained model to make predictions on the test dataset ('tile:xla').\n\n# Create a TileDataset for the 'tile:xla' test dataset\ndataset = TileDataset(tile_xla[\"test\"])\n\n# List to store model predictions for each sample in the test dataset\ntile_xla_predictions = [[] for i in range(len(dataset))]\n\n# Iterate through each fold (previously trained models)\nfor fold in range(6): # 需要与K折保持一致\n    \n    if fold != 0:\n        break\n    \n    # Load the trained model weights for the current fold\n    model.load_state_dict(torch.load(f'/kaggle/working/best_model_{fold}.pth'))\n    model.eval()  # 🕵️ Set the model to evaluation mode\n    pbar = tqdm(range(len(dataset)))  # Progress bar for test data prediction\n    \n    for i in pbar:\n        cfg_ft, nd_ft, nd_op, ind, target = dataset[i]\n        cfg_ft, nd_ft, nd_op, ind, target = cfg_ft.to(device), nd_ft.to(device), nd_op.to(device), ind.to(device), target.to(device)\n\n        out = model(cfg_ft, nd_ft, nd_op, ind)\n        tile_xla_predictions[i].append(out.cpu().detach().numpy())\n\n# Aggregate predictions by taking the mean and selecting the top 5\ntile_xla_predictions = [np.argsort(np.mean(pred, axis=0))[:5] for pred in tile_xla_predictions]\n\n# The 'tile_xla_predictions' now contains the top 5 predicted results for each sample in the 'tile:xla' test dataset.\ntile_xla_predictions\n","metadata":{"papermill":{"duration":42.864288,"end_time":"2023-09-01T19:31:46.765649","exception":false,"start_time":"2023-09-01T19:31:03.901361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:59.367586Z","iopub.status.idle":"2023-09-13T09:35:59.368085Z","shell.execute_reply.started":"2023-09-13T09:35:59.367839Z","shell.execute_reply":"2023-09-13T09:35:59.367863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 📊 Generate and Save Submission File\n# In this section, we generate a submission file based on the model predictions and save it.\n\n# Read the sample submission file\nsub = pd.read_csv('/kaggle/input/predict-ai-model-runtime/sample_submission.csv')\n\n# Iterate through the test file names and update the submission file with top predictions\nfor i, filename in enumerate(tile_xla[\"test\"]['file'].values):\n    id = 'tile:xla:' + filename[:-4]  # Construct the ID for the submission\n    sub.loc[sub.ID == id, 'TopConfigs'] = ';'.join(tile_xla_predictions[i].astype(str))\n\n# Save the updated submission file as 'submission.csv' without the index\nsub.to_csv('submission.csv', index=False)\n\n# Display the updated submission file\nsub\n","metadata":{"papermill":{"duration":20.880172,"end_time":"2023-09-01T19:32:28.307392","exception":false,"start_time":"2023-09-01T19:32:07.42722","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-13T09:35:59.369999Z","iopub.status.idle":"2023-09-13T09:35:59.370498Z","shell.execute_reply.started":"2023-09-13T09:35:59.370260Z","shell.execute_reply":"2023-09-13T09:35:59.370284Z"},"trusted":true},"execution_count":null,"outputs":[]}]}