{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":9735.485807,"end_time":"2023-09-01T19:32:52.176348","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-09-01T16:50:36.690541","version":"2.4.0"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":58266,"databundleVersionId":6641124,"sourceType":"competition"}],"dockerImageVersionId":30527,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Attention!!!\n\nThis is a very simple but bad quality notebook. \n - I do not use any sort of ranking loss, which would be better.\n - My strategy instead is to min-max scale the times and apply L1-loss\n - My model is also not optimized. It is a relatively simple GNN that embeds the graph and only processes 1 datapoint at a time and is only trained on 1 epoch.\n - The public score would be much better if you paired this submission with a trained model for layout. Since this only contributes to half of the score.\n - Have fun playing around with it!\n \n \n # CHANGES\n - V5 - normalized train and infer targets, use MSE loss, changed evaluation metric to perform top5 mean instead of top5 max for robustness, 5-fold CV\n - V6 - use SAGEConv instead of GCN, add dropout layer, increase number of paramters, changed evaluation metric to perform top50 mean, 10->20 epochs.\n - V13 - fixed problem where model weights weren't being reset leading to heavy overfitting...oops","metadata":{"papermill":{"duration":0.009203,"end_time":"2023-09-01T16:50:47.024469","exception":false,"start_time":"2023-09-01T16:50:47.015266","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install torch-geometric torch-scatter","metadata":{"_kg_hide-output":true,"papermill":{"duration":264.245878,"end_time":"2023-09-01T16:55:11.276628","exception":false,"start_time":"2023-09-01T16:50:47.03075","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-09T11:33:11.607044Z","iopub.execute_input":"2023-11-09T11:33:11.607418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom tqdm import tqdm \n\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn\nfrom torch import Tensor\nfrom torch_geometric.nn import GCNConv,SAGEConv\nfrom torch_geometric.datasets import Planetoid\nfrom torch.utils.data import DataLoader, Dataset\nfrom timm.scheduler import CosineLRScheduler\nimport matplotlib.pyplot as plt\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"papermill":{"duration":4.819384,"end_time":"2023-09-01T16:55:16.104784","exception":false,"start_time":"2023-09-01T16:55:11.2854","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_df(directory):\n    splits = [\"train\", \"valid\", \"test\"]\n    dfs = dict()\n    \n    for split in splits:\n        path = os.path.join(directory, split)\n        files = os.listdir(path)\n        list_df = []\n        \n        for file in files:\n            d = dict(np.load(os.path.join(path,file)))\n            d['file'] = file\n            list_df.append(d)\n        dfs[split] = pd.DataFrame.from_dict(list_df)\n    return dfs\ntile_xla = load_df(\"/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/\")","metadata":{"papermill":{"duration":0.020227,"end_time":"2023-09-01T16:55:16.152594","exception":false,"start_time":"2023-09-01T16:55:16.132367","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset and Model","metadata":{"papermill":{"duration":0.008874,"end_time":"2023-09-01T16:56:21.968592","exception":false,"start_time":"2023-09-01T16:56:21.959718","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TileDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        config_feat = torch.tensor(row['config_feat'].astype(np.float32))\n        node_feat = torch.tensor(row['node_feat'].astype(np.float32))\n        node_opcode = torch.tensor(row['node_opcode'].astype(np.int64))\n        edge_index = torch.tensor(np.swapaxes(row['edge_index'],0,1).astype(np.int64))\n        target = (row['config_runtime']/(row['config_runtime_normalizers']+1e-5)).astype(np.float32) #/row['config_runtime_normalizers']\n        # minmax scale the target, we only care about order\n        target = (target-np.mean(target))/(np.std(target)+1e-5)\n\n#         target = (target-np.mean(target))/(np.std(target))\n        target = torch.tensor(target)\n        return config_feat,node_feat,node_opcode,edge_index,target","metadata":{"papermill":{"duration":0.020329,"end_time":"2023-09-01T16:56:21.997734","exception":false,"start_time":"2023-09-01T16:56:21.977405","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleModel(torch.nn.Module):\n    def __init__(self, hidden_channels, graph_in, graph_out, hidden_dim,dropout=0.0):\n        super().__init__()\n        op_embedding_dim = 4 # I choose 4-dimensional embedding\n        self.embedding = torch.nn.Embedding(120, #120 different op-codes\n                                            op_embedding_dim,\n                                           )\n        assert len(hidden_channels)>0\n        \n        self.linear = nn.Linear(op_embedding_dim+140,graph_in)\n        in_channels=graph_in\n        self.convs = torch.nn.ModuleList()\n        last_dim = hidden_channels[0]\n        conv = SAGEConv\n        self.convs.append(conv(in_channels, hidden_channels[0]))\n        for i in range(len(hidden_channels)-1):\n            self.convs.append(conv(hidden_channels[i], hidden_channels[i+1]))\n            last_dim = hidden_channels[i+1]\n        self.convs.append(conv(last_dim, graph_out))\n        \n        \n        \n        self.dense = torch.nn.Sequential(nn.Linear(graph_out*2+24, hidden_dim),\n                                         nn.Dropout(p=dropout),\n                                         nn.ReLU(),\n                                         nn.Linear(hidden_dim, hidden_dim),\n                                         nn.Dropout(p=dropout),\n                                         nn.ReLU(),\n                                         nn.Linear(hidden_dim, 1),\n                                        )\n#         self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, x_cfg: Tensor,x_feat: Tensor, x_op: Tensor, edge_index: Tensor) -> Tensor:\n        \n        #get graph features\n        x = torch.concat([x_feat,self.embedding(x_op)],dim = 1)\n        x = self.linear(x)\n        #pass though conv layers\n        for conv in self.convs:\n            x = conv(x, edge_index).relu()\n        # get 1d graph embedding using average pooling\n        x_mean = x.mean(0)\n        x_max = x.max(0).values\n        \n        #put graph data into config data\n        x = torch.concat([x_cfg,x_max.repeat((len(x_cfg),1)),x_mean.repeat((len(x_cfg),1))],axis=1)\n        #put into dense nn\n        x = torch.flatten(self.dense(x))\n        x = (x-torch.mean(x))/(torch.std(x)+1e-5)\n        return x\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train One Epoch","metadata":{"papermill":{"duration":0.008439,"end_time":"2023-09-01T16:56:22.088164","exception":false,"start_time":"2023-09-01T16:56:22.079725","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.concat((tile_xla[\"train\"],tile_xla[\"valid\"]),axis=0).reset_index(drop=True)","metadata":{"papermill":{"duration":0.021726,"end_time":"2023-09-01T16:56:22.11899","exception":false,"start_time":"2023-09-01T16:56:22.097264","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = sklearn.model_selection.KFold(n_splits=5,shuffle=True,random_state=0)\nscore_means = []\nscore_maxs = []\nfor fold,(tr_idx,va_idx) in enumerate(kfold.split(df)):\n    model = SimpleModel(hidden_channels = [32,48,64,84],graph_in = 64,graph_out = 64,hidden_dim=128,dropout = 0.2).to(device)\n    train_dataset = TileDataset(df.iloc[tr_idx])\n    val_dataset = TileDataset(df.iloc[va_idx])\n    criterion = torch.nn.MSELoss()\n    steps = len(train_dataset)*20\n    warmup_steps = int(steps*0.2)\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4,weight_decay = 1e-4)\n    scheduler = CosineLRScheduler(optimizer,t_initial= steps,warmup_t=warmup_steps, warmup_lr_init=1e-6,lr_min=2e-8,)\n    \n    def score_tile_mean(predictions, df):\n        score = 0\n        for i in range(len(df)):\n            predbest = np.mean(df.iloc[i]['config_runtime'][predictions[i]])\n            best = np.mean(np.sort(df.iloc[i]['config_runtime'])[:50])\n            score += 2-predbest/best\n        score /= len(df)\n        return score\n    def score_tile_max(predictions, df):\n        score = 0\n        for i in range(len(df)):\n            predbest = np.min(df.iloc[i]['config_runtime'][predictions[i][:5]])\n            best = np.min(df.iloc[i]['config_runtime'])\n    #         print(best,predbest)\n            score += 2 - predbest/best\n        score /= len(df)\n        return score\n\n    best_score = 0\n    best_score_max = 0\n    for epoch in range(10):\n        model.train()\n        pbar = tqdm(range(len(train_dataset)),leave=False)\n        loss_sum = 0\n        n = 0\n        for i in pbar:\n            cfg_ft,nd_ft,nd_op,ind,target = train_dataset[i]\n            cfg_ft,nd_ft,nd_op,ind,target = cfg_ft.to(device),nd_ft.to(device),nd_op.to(device),ind.to(device),target.to(device)\n\n            out = model(cfg_ft,nd_ft,nd_op,ind)\n            loss = criterion(out, target)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1e-2)\n            scheduler.step(i+len(train_dataset)*epoch)\n            optimizer.step()\n            loss_sum+=loss.item()\n            n+=1\n            pbar.set_description(f'running loss: {(loss_sum/n):.2f},current loss: {(loss.item()):.2f}')\n        pbar.close()\n        model.eval()\n\n        tile_xla_predictions = []\n        pbar = tqdm(range(len(val_dataset)),leave=False)\n        for i in pbar:\n            cfg_ft,nd_ft,nd_op,ind,target = val_dataset[i]\n            cfg_ft,nd_ft,nd_op,ind,target = cfg_ft.to(device),nd_ft.to(device),nd_op.to(device),ind.to(device),target.to(device)\n\n            out = model(cfg_ft,nd_ft,nd_op,ind)\n            tile_xla_predictions.append(np.argsort(out.cpu().detach().numpy())[:50])\n        pbar.close()\n        score_mean = score_tile_mean(tile_xla_predictions, val_dataset.df)\n        score_max = score_tile_max(tile_xla_predictions, val_dataset.df)\n        print(f'fold {fold} epoch {epoch}, comp_score = {score_max:.3f}, mean_score = {score_mean:.3f},')\n        if score_mean>best_score:\n            best_score = score_mean\n            best_score_max = score_max\n            torch.save(model.state_dict(), f'best_model_{fold}.pth')\n    score_means.append(best_score)\n    score_maxs.append(best_score_max)\nprint(f'comp_score = {np.mean(score_maxs)}, mean_score = {np.mean(score_means)},')","metadata":{"papermill":{"duration":9138.052878,"end_time":"2023-09-01T19:28:40.180685","exception":false,"start_time":"2023-09-01T16:56:22.127807","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate on Validation Dataset","metadata":{"papermill":{"duration":20.903162,"end_time":"2023-09-01T19:29:21.465429","exception":false,"start_time":"2023-09-01T19:29:00.562267","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**0.31 is not bad considering that this model only trained on 1 epoch and is not on a ranking loss!**","metadata":{"papermill":{"duration":20.540632,"end_time":"2023-09-01T19:30:02.340577","exception":false,"start_time":"2023-09-01T19:29:41.799945","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Predict and Submit (only tile:xla predictions)","metadata":{"papermill":{"duration":20.552961,"end_time":"2023-09-01T19:30:43.664106","exception":false,"start_time":"2023-09-01T19:30:23.111145","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dataset = TileDataset(tile_xla[\"test\"])\ntile_xla_predictions = [[] for i in range(len(dataset))]\nfor fold in range(5):\n    model.load_state_dict(torch.load(f'/kaggle/working/best_model_{fold}.pth'))\n    model.eval()\n    pbar = tqdm(range(len(dataset)))\n    for i in pbar:\n        cfg_ft,nd_ft,nd_op,ind,target = dataset[i]\n        cfg_ft,nd_ft,nd_op,ind,target = cfg_ft.to(device),nd_ft.to(device),nd_op.to(device),ind.to(device),target.to(device)\n\n        out = model(cfg_ft,nd_ft,nd_op,ind)\n        tile_xla_predictions[i].append(out.detach().numpy())\ntile_xla_predictions = [np.argsort(np.mean(pred,axis=0))[:5] for pred in tile_xla_predictions]","metadata":{"papermill":{"duration":42.864288,"end_time":"2023-09-01T19:31:46.765649","exception":false,"start_time":"2023-09-01T19:31:03.901361","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/predict-ai-model-runtime/sample_submission.csv')\nfor i,filename in enumerate(tile_xla[\"test\"]['file'].values):\n    id = 'tile:xla:' +filename[:-4]\n    sub.loc[sub.ID == id,'TopConfigs'] = ';'.join(tile_xla_predictions[i].astype(str))\nsub.to_csv('submission.csv',index=False)\nsub","metadata":{"papermill":{"duration":20.880172,"end_time":"2023-09-01T19:32:28.307392","exception":false,"start_time":"2023-09-01T19:32:07.42722","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-09T11:30:01.399743Z","iopub.status.idle":"2023-11-09T11:30:01.400113Z","shell.execute_reply.started":"2023-11-09T11:30:01.399911Z","shell.execute_reply":"2023-11-09T11:30:01.399927Z"},"trusted":true},"execution_count":null,"outputs":[]}]}