{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Install | Import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport torch\nfrom torch import optim\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:33:46.614442Z","iopub.execute_input":"2023-10-07T14:33:46.615183Z","iopub.status.idle":"2023-10-07T14:33:51.564701Z","shell.execute_reply.started":"2023-10-07T14:33:46.615138Z","shell.execute_reply":"2023-10-07T14:33:51.562667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = torch.device('cuda')\nelse:\n    device = torch.device('cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:33:51.566920Z","iopub.execute_input":"2023-10-07T14:33:51.567624Z","iopub.status.idle":"2023-10-07T14:33:51.579327Z","shell.execute_reply.started":"2023-10-07T14:33:51.567581Z","shell.execute_reply":"2023-10-07T14:33:51.577864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data | Pipeline (Generalized)","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input/'):\n    if len(filenames) != 0:\n        if filenames[0] != \"sample_submission.csv\":\n            avg = np.array([os.path.getsize(os.path.join(dirname, filename)) for filename in filenames]).mean()\n            # os.path.getsize returns the size of the dictionary passed in bytes\n            print(dirname, len(os.listdir(dirname)))\n            print(\"Size: {:.3f} KB\".format(avg/1024)) # 1024 is 1KB","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:33:51.581297Z","iopub.execute_input":"2023-10-07T14:33:51.581746Z","iopub.status.idle":"2023-10-07T14:34:04.260275Z","shell.execute_reply.started":"2023-10-07T14:33:51.581706Z","shell.execute_reply":"2023-10-07T14:34:04.259142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tile = np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/train/retinanet.4x4.fp32_-431a58cc30e72ec6.npz')\ntile.files","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:34:04.262999Z","iopub.execute_input":"2023-10-07T14:34:04.264125Z","iopub.status.idle":"2023-10-07T14:34:04.279936Z","shell.execute_reply.started":"2023-10-07T14:34:04.264089Z","shell.execute_reply":"2023-10-07T14:34:04.278140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basic_structure = tile.files\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    flag = False\n    for filename in filenames:\n        if filename != \"sample_submission.csv\" and filename[-4:] == '.npz':\n            \n            if np.load(os.path.join(dirname, filename)).files != basic_structure and not flag:\n                print(dirname)\n                print(np.load(os.path.join(dirname, filename)).files)\n                flag = True","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:34:04.282003Z","iopub.execute_input":"2023-10-07T14:34:04.282632Z","iopub.status.idle":"2023-10-07T14:35:05.771399Z","shell.execute_reply.started":"2023-10-07T14:34:04.282589Z","shell.execute_reply":"2023-10-07T14:35:05.770569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(directory):\n    splits = ['train', 'valid', 'test']\n    dfs = dict()\n    for split in splits:\n        path = os.path.join(directory, split)\n        files = os.listdir(path)\n        list_df = []\n        \n        for file in files:\n            list_df.append(dict(np.load(os.path.join(path, file))))\n        dfs[split] = pd.DataFrame.from_dict(list_df)\n    return dfs","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.772901Z","iopub.execute_input":"2023-10-07T14:35:05.773751Z","iopub.status.idle":"2023-10-07T14:35:05.779795Z","shell.execute_reply.started":"2023-10-07T14:35:05.773719Z","shell.execute_reply":"2023-10-07T14:35:05.778594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocess datum\ndef make_datum(datum, i):\n    configs = datum.config_feat[i].shape[0]\n    graph = pd.DataFrame(columns=['node_feat', 'node_opcode', 'edge_index', 'config_feat', 'config_runtime', 'config_runtime_normalizers'])\n    for config in range(0, configs):\n        Sample = pd.Series()\n        Sample['node_feat'] = datum.node_feat[i]\n        Sample['node_opcode'] = datum.node_opcode[i]\n        Sample['edge_index'] = datum.edge_index[i]\n        Sample['config_feat'] = datum.config_feat[i][config]\n        Sample['config_runtime'] = datum.config_runtime[i][config]\n        Sample['config_runtime_normalizers'] = datum.config_runtime_normalizers[i][config]\n        Sample['topo_order'] = datum.topo_order.values[0]\n        graph = pd.concat([graph, Sample.to_frame().T], ignore_index=True)\n    return graph","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.781233Z","iopub.execute_input":"2023-10-07T14:35:05.781791Z","iopub.status.idle":"2023-10-07T14:35:05.801331Z","shell.execute_reply.started":"2023-10-07T14:35:05.781752Z","shell.execute_reply":"2023-10-07T14:35:05.800138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocess data\ndef make_data(data, b=0):\n    df = pd.DataFrame(columns = data.columns)\n    for i in range(len(data)):\n        datum = data.iloc[i].to_frame().T\n        graph = make_datum(datum, i+b)\n        df = pd.concat([df, graph], axis=0, ignore_index=True)\n    df['avg_runtime'] = df['config_runtime'] / (df['config_runtime_normalizers'] + 1e-5)\n    df.drop(['config_runtime', 'config_runtime_normalizers'], axis=1, inplace=True)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.803030Z","iopub.execute_input":"2023-10-07T14:35:05.803411Z","iopub.status.idle":"2023-10-07T14:35:05.825456Z","shell.execute_reply.started":"2023-10-07T14:35:05.803328Z","shell.execute_reply":"2023-10-07T14:35:05.824258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_loader to load data in batchs\nclass dataset(Dataset):\n    def __init__(self, data):\n        self.feats = data[data.columns[:-1]]\n        self.labels = data[data.columns[-1]]\n        self.transform = transforms.Compose([transforms.ToTensor()])\n    \n    def __len__(self):\n        return len(self.feats)\n    \n    def __getitem__(self, idx):\n        # early join of config features\n        op_code = torch.tensor(self.feats.node_opcode.iloc[idx])\n        node_f = torch.tensor(self.feats.node_feat.iloc[idx])\n        config_f = torch.tensor(self.feats.config_feat.iloc[idx])\n        adj_mat = torch.tensor(self.feats.edge_index.iloc[idx])\n        topo_o = torch.tensor(self.feats.topo_order.iloc[idx])\n        config_f_broadcasted = config_f.unsqueeze(0).expand(node_f.size(0), -1)\n        node_config_f = torch.cat((node_f, config_f_broadcasted), axis=1)\n        # padding for compatibility with the model GNN\n        to_add_rows = 500 - node_config_f.size(0)\n        node_config_row_zeros = torch.zeros((to_add_rows, node_config_f.size(1)), dtype=node_config_f.dtype)\n        node_config_f = torch.cat((node_config_f, node_config_row_zeros), dim=0)\n        op_top_zeros = torch.zeros(to_add_rows, dtype=op_code.dtype)\n        op_code = torch.cat((op_code, op_top_zeros))\n        topo_o = torch.cat((topo_o, op_top_zeros))\n        adj_row_zeros = torch.zeros((to_add_rows, adj_mat.size(1)), dtype=adj_mat.dtype)\n        adj_mat = torch.cat((adj_mat, adj_row_zeros), dim=0)\n        adj_col_zeros = torch.zeros((adj_mat.size(0), to_add_rows), dtype=adj_mat.dtype)\n        adj_mat = torch.cat((adj_mat, adj_col_zeros), dim=1)\n        # padding done (nodes == 100)\n        inputs = [op_code, node_config_f, adj_mat, topo_o]\n        return inputs, torch.tensor(self.labels.iloc[idx])","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.827119Z","iopub.execute_input":"2023-10-07T14:35:05.827490Z","iopub.status.idle":"2023-10-07T14:35:05.842104Z","shell.execute_reply.started":"2023-10-07T14:35:05.827455Z","shell.execute_reply":"2023-10-07T14:35:05.841225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test tile loader\n# made_tilex_train = make_data(tilex_train.head(1))\ntest = False\nif(test):\n    custom_dataset = dataset(made_tilex_train)\n    loader = DataLoader(custom_dataset, batch_size=2, shuffle=True)\n    for batch in loader:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.845370Z","iopub.execute_input":"2023-10-07T14:35:05.845665Z","iopub.status.idle":"2023-10-07T14:35:05.864930Z","shell.execute_reply.started":"2023-10-07T14:35:05.845642Z","shell.execute_reply":"2023-10-07T14:35:05.863422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test model gnn using batch size 2\ntest = False\nif(test):\n    gnn = GNN(174)\n    batch_feats, batch_true = batch\n    batch_preds = gnn(batch_feats)\n    print(batch_preds)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.866871Z","iopub.execute_input":"2023-10-07T14:35:05.867334Z","iopub.status.idle":"2023-10-07T14:35:05.882467Z","shell.execute_reply.started":"2023-10-07T14:35:05.867296Z","shell.execute_reply":"2023-10-07T14:35:05.881235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Helper functions","metadata":{}},{"cell_type":"code","source":"def adj_mat(edges, n):\n    # edges are 0-indexed\n    adj_matrix = np.zeros((n, n))\n    for u, v in edges:\n        adj_matrix[u][v] = 1\n        adj_matrix[v][u] = 1\n    return adj_matrix","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.883302Z","iopub.execute_input":"2023-10-07T14:35:05.883563Z","iopub.status.idle":"2023-10-07T14:35:05.906187Z","shell.execute_reply.started":"2023-10-07T14:35:05.883541Z","shell.execute_reply":"2023-10-07T14:35:05.905058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test adj_mat\nadj_matrix = adj_mat([[2, 3], [3, 1]], 4)\nprint(type(adj_matrix))\nadj_matrix","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.909123Z","iopub.execute_input":"2023-10-07T14:35:05.909472Z","iopub.status.idle":"2023-10-07T14:35:05.928177Z","shell.execute_reply.started":"2023-10-07T14:35:05.909445Z","shell.execute_reply":"2023-10-07T14:35:05.927059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def topo_sort(edges):\n    indeg = {}\n    for edge in edges:\n        if(edge[0] not in indeg):\n            indeg[edge[0]] = 0\n        if(edge[1] not in indeg):\n            indeg[edge[1]] = 0\n        indeg[edge[1]] += 1\n    \n    queue = []\n    for node in indeg:\n        if(indeg[node] == 0):\n            queue.append(node)\n    topo_order = []\n    while(queue):\n        node = queue.pop()\n        topo_order.append(node)\n        for neighbour in [edge[1] for edge in edges if edge[0] == node]:\n            indeg[neighbour] -= 1\n            if(indeg[neighbour] == 0): \n                queue.append(neighbour)\n    # let's reorder the topological sort to be compatible with topo-order aware downsampling\n    topo_embd = []\n    for i, node in enumerate(topo_order):\n        topo_embd.append((node, i))\n    topo_embd = sorted(topo_embd)\n    topo_order = []\n    for node, i in topo_embd:\n        topo_order.append(i)\n    return topo_order","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.929493Z","iopub.execute_input":"2023-10-07T14:35:05.929782Z","iopub.status.idle":"2023-10-07T14:35:05.942495Z","shell.execute_reply.started":"2023-10-07T14:35:05.929759Z","shell.execute_reply":"2023-10-07T14:35:05.941454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test topo_sort\nedges = [[2, 3], [3, 1], [1, 4], [3, 4]]\ntopo_sort(edges)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.943835Z","iopub.execute_input":"2023-10-07T14:35:05.944256Z","iopub.status.idle":"2023-10-07T14:35:05.959585Z","shell.execute_reply.started":"2023-10-07T14:35:05.944221Z","shell.execute_reply":"2023-10-07T14:35:05.958371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def kipf_norm(adj_mats):\n    norm_mats = []\n    for adj_mat in adj_mats:\n        max_degree = torch.max(torch.sum(adj_mat, dim=1))\n        # add inv-deg term if convergence is slow\n        norm_mat = adj_mat / max_degree\n        norm_mats.append(norm_mat)\n    return torch.stack(norm_mats, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.962101Z","iopub.execute_input":"2023-10-07T14:35:05.962397Z","iopub.status.idle":"2023-10-07T14:35:05.973392Z","shell.execute_reply.started":"2023-10-07T14:35:05.962373Z","shell.execute_reply":"2023-10-07T14:35:05.972552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Tile Config","metadata":{}},{"cell_type":"code","source":"tile_xla = load_data('/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/')","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:05.974404Z","iopub.execute_input":"2023-10-07T14:35:05.974788Z","iopub.status.idle":"2023-10-07T14:35:36.318598Z","shell.execute_reply.started":"2023-10-07T14:35:05.974755Z","shell.execute_reply":"2023-10-07T14:35:36.317398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tilex_train = tile_xla['train']\ntilex_valid = tile_xla['valid']\ntilex_test = tile_xla['test']","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.320283Z","iopub.execute_input":"2023-10-07T14:35:36.320578Z","iopub.status.idle":"2023-10-07T14:35:36.324935Z","shell.execute_reply.started":"2023-10-07T14:35:36.320554Z","shell.execute_reply":"2023-10-07T14:35:36.323832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tilex_train.shape)\n# convert all the edge_index to adjacency matrix\n# also add topo_order of edge_index\ntilex_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.326417Z","iopub.execute_input":"2023-10-07T14:35:36.326800Z","iopub.status.idle":"2023-10-07T14:35:36.724220Z","shell.execute_reply.started":"2023-10-07T14:35:36.326774Z","shell.execute_reply":"2023-10-07T14:35:36.722723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare train data 1\ntest = False\nif test:\n    tile_train = tilex_train.head(len(tilex_train)//2).copy(deep=True)\n    tile_train['topo_order'] = tile_train['edge_index'].map(lambda e: topo_sort(e))\n    tile_train['edge_index'] = tile_train['edge_index'].map(lambda e: adj_mat(e, max([max(edge) for edge in e])+1))\n    tile_train = make_data(tile_train)\n    tile_train.to_pickle('/kaggle/working/tiler_train1.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.725698Z","iopub.execute_input":"2023-10-07T14:35:36.726095Z","iopub.status.idle":"2023-10-07T14:35:36.733331Z","shell.execute_reply.started":"2023-10-07T14:35:36.726064Z","shell.execute_reply":"2023-10-07T14:35:36.731384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare train data 2\ntest = False\nif test:\n    b = len(tilex_train)//2\n    tile_train = tilex_train.tail(len(tilex_train)-b).copy(deep=True)\n    tile_train['topo_order'] = tile_train['edge_index'].map(lambda e: topo_sort(e))\n    tile_train['edge_index'] = tile_train['edge_index'].map(lambda e: adj_mat(e, max([max(edge) for edge in e])+1))\n    tile_train = make_data(tile_train, b)\n    tile_train.to_pickle('/kaggle/working/tiler_train2.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.734437Z","iopub.execute_input":"2023-10-07T14:35:36.734717Z","iopub.status.idle":"2023-10-07T14:35:36.748915Z","shell.execute_reply.started":"2023-10-07T14:35:36.734695Z","shell.execute_reply":"2023-10-07T14:35:36.747783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare valid data\ntest = False\nif test:\n    tile_val = tilex_valid.copy(deep=True)\n    tile_val['topo_order'] = tile_val['edge_index'].map(lambda e: topo_sort(e))\n    tile_val['edge_index'] = tile_val['edge_index'].map(lambda e: adj_mat(e, max([max(edge) for edge in e])+1))\n    tile_val = make_data(tile_val)\n    tile_val.to_pickle('/kaggle/working/tiler_val.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.750364Z","iopub.execute_input":"2023-10-07T14:35:36.750694Z","iopub.status.idle":"2023-10-07T14:35:36.764374Z","shell.execute_reply.started":"2023-10-07T14:35:36.750669Z","shell.execute_reply":"2023-10-07T14:35:36.763457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test GNN, gConv, data preprocessing steps\ntoy = tilex_train.head(1).copy(deep=True)\nn = toy.node_feat[0].shape[0]\ntoy['topo_order'] = toy['edge_index'].map(lambda e: topo_sort(e))","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:35:36.765696Z","iopub.execute_input":"2023-10-07T14:35:36.765977Z","iopub.status.idle":"2023-10-07T14:35:36.782292Z","shell.execute_reply.started":"2023-10-07T14:35:36.765932Z","shell.execute_reply":"2023-10-07T14:35:36.781444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# edge_index -> adj_matrix\ntoy['edge_index'] = toy['edge_index'].map(lambda e: adj_mat(e, n))\ntoy","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:52:53.073637Z","iopub.execute_input":"2023-10-07T14:52:53.073980Z","iopub.status.idle":"2023-10-07T14:52:53.161185Z","shell.execute_reply.started":"2023-10-07T14:52:53.073931Z","shell.execute_reply":"2023-10-07T14:52:53.160221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(toy.edge_index[0].shape)\ntoy","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:52:54.675583Z","iopub.execute_input":"2023-10-07T14:52:54.675944Z","iopub.status.idle":"2023-10-07T14:52:54.763449Z","shell.execute_reply.started":"2023-10-07T14:52:54.675915Z","shell.execute_reply":"2023-10-07T14:52:54.762273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# apply GNN, gConv on toy\nimport warnings\nwarnings.filterwarnings('ignore')\nmade_toy = make_data(toy)\nmade_toy","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:20.264411Z","iopub.execute_input":"2023-10-07T14:54:20.264762Z","iopub.status.idle":"2023-10-07T14:54:27.599992Z","shell.execute_reply.started":"2023-10-07T14:54:20.264737Z","shell.execute_reply":"2023-10-07T14:54:27.599020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make data into batchs and test it on gConv and GNN\ncustom_dataset = dataset(made_toy)\nloader = DataLoader(custom_dataset, batch_size=2, shuffle=True)\nfor batch in loader:\n    break\nbatch","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:30.934059Z","iopub.execute_input":"2023-10-07T14:54:30.934437Z","iopub.status.idle":"2023-10-07T14:54:31.122669Z","shell.execute_reply.started":"2023-10-07T14:54:30.934407Z","shell.execute_reply":"2023-10-07T14:54:31.121166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use batch to test the model\ntest = False\nif test:\n    batch_feat, batch_labels = batch\n    model = GNN(174, 2)\n    model = model.to(device)\n    batch_preds = model(batch_feat)\n    batch_preds","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:31.136110Z","iopub.execute_input":"2023-10-07T14:54:31.136522Z","iopub.status.idle":"2023-10-07T14:54:31.143176Z","shell.execute_reply.started":"2023-10-07T14:54:31.136483Z","shell.execute_reply":"2023-10-07T14:54:31.141785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = np.array([os.path.getsize(os.path.join(dirname, filename)) for filename in filenames]).mean()\n# what does this mean, something related to the size of the data?","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:31.334055Z","iopub.execute_input":"2023-10-07T14:54:31.334397Z","iopub.status.idle":"2023-10-07T14:54:31.348543Z","shell.execute_reply.started":"2023-10-07T14:54:31.334368Z","shell.execute_reply":"2023-10-07T14:54:31.347621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's deal with one sample/computation graph\nsample = tilex_train.head(1)\nprint('no of nodes: (#nodes) in computation graph ', len(sample.node_feat[0]))\nprint('node_feat: (#features_of_a_node) ', len(sample.node_feat[0][0]))\nprint('node_opcode: (#operation_codes) ', len(sample.node_opcode[0]))\nprint('edge_index: (#edges) ', len(sample.edge_index[0]))\nprint('configs: (#configurations) ', len(sample.config_feat[0]))\nprint('config_feat: (#config_features) ', len(sample.config_feat[0][0]))\nprint('config_runtime: (#runtimes == # configs) ', len(sample.config_runtime[0]))\nprint('config_runtime_normalizers: (#runtimes_normalized == #configs) ', len(sample.config_runtime_normalizers[0]))","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:31.535131Z","iopub.execute_input":"2023-10-07T14:54:31.535795Z","iopub.status.idle":"2023-10-07T14:54:31.544598Z","shell.execute_reply.started":"2023-10-07T14:54:31.535764Z","shell.execute_reply":"2023-10-07T14:54:31.543316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's check another example\nsample = tilex_train.head(2)\nprint('no of nodes: (#nodes) in computation graph ', len(sample.node_feat[1]))\nprint('node_feat: (#features_of_a_node) ', len(sample.node_feat[1][0]))\nprint('node_opcode: (#operation_codes) ', len(sample.node_opcode[1]))\nprint('edge_index: (#edges) ', len(sample.edge_index[1]))\nprint('configs: (#configurations) ', len(sample.config_feat[1]))\nprint('config_feat: (#config_features) ', len(sample.config_feat[1][0]))\nprint('config_runtime: (#runtimes == # configs) ', len(sample.config_runtime[1]))\nprint('config_runtime_normalizers: (#runtimes_normalized == #configs) ', len(sample.config_runtime_normalizers[1]))","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:31.735247Z","iopub.execute_input":"2023-10-07T14:54:31.735590Z","iopub.status.idle":"2023-10-07T14:54:31.743646Z","shell.execute_reply.started":"2023-10-07T14:54:31.735565Z","shell.execute_reply":"2023-10-07T14:54:31.742825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class gConv(nn.Module):\n    def __init__(self, in_feat, out_feat, batch_size):\n        super(gConv, self).__init__()\n        self.in_feat = in_feat\n        self.out_feat = out_feat\n        # try nn.init xavier_normal_ or xavier_uniform_ if convergence is slow\n        self.w = nn.Parameter(torch.stack([torch.randn(self.in_feat, self.out_feat, device=device)]*batch_size, axis=0), requires_grad=True)\n        \n    def forward(self, adj_matrix, feats):\n        adj_matrix = adj_matrix.to(device)\n        feats = feats.to(device)\n        id_matrix = torch.eye(feats.shape[1], device=device)\n        id_matrix = torch.stack([id_matrix]*feats.shape[0], axis=0)\n        adj_matrix = adj_matrix + id_matrix\n        adj_matrix = kipf_norm(adj_matrix)\n        agg_feat = torch.matmul(torch.matmul(adj_matrix, feats), self.w)\n        return torch.relu(agg_feat)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:31.936253Z","iopub.execute_input":"2023-10-07T14:54:31.936642Z","iopub.status.idle":"2023-10-07T14:54:31.944291Z","shell.execute_reply.started":"2023-10-07T14:54:31.936613Z","shell.execute_reply":"2023-10-07T14:54:31.943329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test gConv\ngconv = gConv(in_feat=3, out_feat=2, batch_size=2) # feats in, feats out\ngconv = gconv.to(device) # need re-assign\nadj_matrix = torch.FloatTensor([[0, 1, 1], \n              [1, 0, 0], \n              [1, 0, 0]])\nfeats = torch.FloatTensor([[1, 2, 3], \n                           [2, 1, 2], \n                           [3, 1, 5]])\nx = gconv(torch.unsqueeze(adj_matrix, 0), torch.unsqueeze(feats, 0))\nx\n# upgrade gConv object to cuda and pass tensors on cuda while calling\n# compatible to batchsize","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:32.115398Z","iopub.execute_input":"2023-10-07T14:54:32.115778Z","iopub.status.idle":"2023-10-07T14:54:32.147665Z","shell.execute_reply.started":"2023-10-07T14:54:32.115749Z","shell.execute_reply":"2023-10-07T14:54:32.146589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GNN(nn.Module):\n    def __init__(self, input_feat, batch_size):\n        super(GNN, self).__init__()\n        self.input_feat = input_feat\n        self.gConv1 = gConv(input_feat, 256, batch_size)\n        self.gConv1.to(device)\n        self.gConv2 = gConv(256, 512, batch_size)\n        self.gConv2.to(device)\n        self.gConv3 = gConv(512, 128, batch_size)\n        self.gConv3.to(device)\n        self.gConv4 = gConv(128, 64, batch_size)\n        self.gConv4.to(device)\n        self.gConv5 = gConv(64, 32, batch_size)\n        self.gConv5.to(device)\n    \n    def forward(self, inputs):\n        op_code, node_config_feat, adj_matrix, topo_order = inputs\n        op_code, node_config_feat, adj_matrix, topo_order = op_code, node_config_feat, adj_matrix, topo_order\n        embd_layer = nn.Embedding(120, 10)\n        # type casting\n        op_code = op_code.clone().detach().to(torch.int64)\n        adj_matrix = adj_matrix.clone().detach().to(torch.float)\n        topo_order = topo_order.clone().detach().to(torch.int64)\n        # type casting\n        op_embd = embd_layer(op_code)\n        feats = torch.cat((op_embd, node_config_feat), dim=-1)\n        # type casting\n        feats = feats.clone().detach().to(torch.float)\n        # type casting\n        assert (feats.shape[-1] == (10+140+24))\n        # graph convolutions\n        x = self.gConv1(adj_matrix, feats)\n        x = self.gConv2(adj_matrix, x)\n        x = self.gConv3(adj_matrix, x)\n        x = self.gConv4(adj_matrix, x)\n        x = self.gConv5(adj_matrix, x)\n        assert(x.shape[1] == feats.shape[1])\n        assert(x.shape[2] == 32)\n        # topological order aware downsampling\n        topo_embd_layer = nn.Embedding(feats.shape[1], 10)\n        topo_embd = topo_embd_layer(topo_order)\n        topo_embd = torch.transpose(topo_embd, 1, 2)\n        assert(topo_embd.shape[1] == 10)\n        assert(topo_embd.shape[2] == feats.shape[1])\n        latent_feat = torch.matmul(topo_embd.to(device), x.to(device))\n        flat_feat = torch.flatten(latent_feat, start_dim=1)\n        flat_dim = flat_feat.shape[1]\n        \n        # linear layers init\n        self.dense1 = nn.Linear(flat_dim, 256).to(device)\n        self.bn1 = nn.BatchNorm1d(256).to(device)\n        self.dense2 = nn.Linear(256, 128).to(device)\n        self.bn2 = nn.BatchNorm1d(128).to(device)\n        self.dense3 = nn.Linear(128, 32).to(device)\n        self.bn3 = nn.BatchNorm1d(32).to(device)\n        self.dense4 = nn.Linear(32, 8).to(device)\n        self.bn4 = nn.BatchNorm1d(8).to(device)\n        self.dense5 = nn.Linear(8, 1).to(device)\n        self.act = nn.ReLU().to(device)\n        # linear layers build\n        x = self.act(self.dense1(flat_feat))\n        x = self.bn1(x)\n        x = self.act(self.dense2(x))\n        x = self.bn2(x)\n        x = self.act(self.dense3(x))\n        x = self.bn3(x)\n        x = self.act(self.dense4(x))\n        x = self.bn4(x)\n        x = self.act(self.dense5(x))\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:32.340592Z","iopub.execute_input":"2023-10-07T14:54:32.341257Z","iopub.status.idle":"2023-10-07T14:54:32.356839Z","shell.execute_reply.started":"2023-10-07T14:54:32.341210Z","shell.execute_reply":"2023-10-07T14:54:32.355167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train Tile from preprocessed data","metadata":{}},{"cell_type":"code","source":"# ensure that batch_size is divisble by train data len and valid data len\nbatch_size = 64 # determine the batch_size closely (also frame a algorithmic question related to this and post in on LeetCode)\nmodel = GNN(174, batch_size)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:32.745145Z","iopub.execute_input":"2023-10-07T14:54:32.745472Z","iopub.status.idle":"2023-10-07T14:54:32.795579Z","shell.execute_reply.started":"2023-10-07T14:54:32.745448Z","shell.execute_reply":"2023-10-07T14:54:32.794401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1_pkl = \"/kaggle/input/processed-tile-data-1/tiler_train1.pkl\"\ntrain2_pkl = \"/kaggle/input/processed-tile-data-2/tiler_train2.pkl\"\nval_pkl = \"/kaggle/input/valid-fast-slow/tiler_val.pkl\"\ntrain1 = True\nif train1:\n    tile_train = pd.read_pickle(train1_pkl)\nelse:\n    tile_train = pd.read_pickle(train2_pkl)\ntile_val = pd.read_pickle(val_pkl)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:54:32.934988Z","iopub.execute_input":"2023-10-07T14:54:32.935347Z","iopub.status.idle":"2023-10-07T14:55:12.394193Z","shell.execute_reply.started":"2023-10-07T14:54:32.935322Z","shell.execute_reply":"2023-10-07T14:55:12.392360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train data rows: \", len(tile_train))\n# slice the data to make it compatible with the batch_size\ntrain_just = tile_train.iloc[0:5546240].copy(deep=True)\ntrain_just.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:12.396148Z","iopub.execute_input":"2023-10-07T14:55:12.396516Z","iopub.status.idle":"2023-10-07T14:55:15.178127Z","shell.execute_reply.started":"2023-10-07T14:55:12.396488Z","shell.execute_reply":"2023-10-07T14:55:15.177051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"valid data rows: \", len(tile_val))\n# slice the data to make it compatible with the batch_size\nval_just = tile_val.iloc[0:1042048].copy(deep=True)\nval_just.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:15.179505Z","iopub.execute_input":"2023-10-07T14:55:15.180512Z","iopub.status.idle":"2023-10-07T14:55:16.132345Z","shell.execute_reply.started":"2023-10-07T14:55:15.180462Z","shell.execute_reply":"2023-10-07T14:55:16.131053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training model\ntest = True\nif test:\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.01)\n    num_epochs = 0 # change the no of epochs to train it\n    train_batchs = dataset(train_just)\n    val_batchs = dataset(val_just)\n    train_batchs = DataLoader(train_batchs, batch_size=batch_size, shuffle=True)\n    val_batchs = DataLoader(val_batchs, batch_size=batch_size, shuffle=True) \n    for epoch in range(num_epochs):\n        tot_loss = 20\n        for batch in train_batchs:\n            batch_feats, batch_true = batch\n            optimizer.zero_grad()\n            batch_preds = model(batch_feats)\n            batch_preds = batch_preds.view(batch_size,)\n            loss = criterion(batch_preds.to(device), batch_true.to(device))\n            tot_loss += loss.item()\n            # add high weightage to top 5 fastest runtime config (ranking loss)\n            loss.backward()\n            optimizer.step()\n        print('train tot_loss: ', tot_loss, end='  ')\n        tot_loss = 0\n        with torch.no_grad():\n            for batch in val_batchs:\n                batch_feats, batch_true = batch\n                batch_preds = model(batch_feats)\n                batch_preds = batch_preds.view(batch_size,)\n                tot_loss += criterion(batch_preds.to(device), batch_true.to(device)).item()\n        print('valid tot_loss: ', tot_loss)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:16.134944Z","iopub.execute_input":"2023-10-07T14:55:16.135288Z","iopub.status.idle":"2023-10-07T14:55:16.633732Z","shell.execute_reply.started":"2023-10-07T14:55:16.135262Z","shell.execute_reply":"2023-10-07T14:55:16.632252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the model\ncheckpoint = {\n    'model_state_dict': model.state_dict(),\n    'optimizer_state_dict': optimizer.state_dict(),\n}\nmodel_path = '/kaggle/working/trained_model.pth'\ntorch.save(checkpoint, model_path)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:16.636009Z","iopub.execute_input":"2023-10-07T14:55:16.636440Z","iopub.status.idle":"2023-10-07T14:55:16.752970Z","shell.execute_reply.started":"2023-10-07T14:55:16.636409Z","shell.execute_reply":"2023-10-07T14:55:16.752076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the model\nload = False\nif load:\n    model = GNN()\n    checkpoint = torch.load(model_path)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    optimizer = optim.Adam(model.parameters(), lr=0.01)\n    optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:16.754780Z","iopub.execute_input":"2023-10-07T14:55:16.755461Z","iopub.status.idle":"2023-10-07T14:55:16.761743Z","shell.execute_reply.started":"2023-10-07T14:55:16.755420Z","shell.execute_reply":"2023-10-07T14:55:16.760620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare the testing data\ntest_data = tile_xla[\"test\"]\nprint(len(test_data))\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:16.763418Z","iopub.execute_input":"2023-10-07T14:55:16.763899Z","iopub.status.idle":"2023-10-07T14:55:17.084291Z","shell.execute_reply.started":"2023-10-07T14:55:16.763870Z","shell.execute_reply":"2023-10-07T14:55:17.083418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False: # Change it to True to run the cell\n    model.eval()\n    result = []\n    for i in range(2): # len(test_data)\n        row = test_data.iloc[i].to_frame().T\n        row = row.copy(deep=True)\n        row['topo_order'] = row['edge_index'].map(lambda e: topo_sort(e))\n        row['edge_index'] = row['edge_index'].map(lambda e: adj_mat(e, max([max(edge) for edge in e])+1))\n        made_row = make_data(row, i)\n        rank = []\n        for config in range(2): # len(made_row)\n            econ = made_row.iloc[config].to_frame().T\n            econ_batch = dataset(econ)\n            econ_data = DataLoader(econ_batch, batch_size=1, shuffle=False)\n            for batch in econ_data:\n                batch_feats, batch_labels = batch\n                pred = model(batch_feats)\n                pred_mean = torch.mean(pred).item()\n                print(pred_mean, batch_labels.item())\n            rank.append((config, pred_mean))\n        rank.sort(key=lambda x: x[1])\n        # take top 5 and append to sample_submissions.csv\n        result.append([x[0] for x in rank[:5]])","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:17.085558Z","iopub.execute_input":"2023-10-07T14:55:17.086223Z","iopub.status.idle":"2023-10-07T14:55:17.094990Z","shell.execute_reply.started":"2023-10-07T14:55:17.086190Z","shell.execute_reply":"2023-10-07T14:55:17.093793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add result to sample_submissions.csv to submit the file of predictions\n# join each inner list with commas and add it to corresponding id row in the sample_submissions.csv\n\nif False: # Change it to True to get the inference\n    assert(len(result) == 844)\n    sub = pd.read_csv(\"/kaggle/input/predict-ai-model-runtime/sample_submission.csv\")\n    for i, res in enumerate(result):\n        s = \"\"\n        for config in res:\n            s += str(config)+';'\n        s = s[:-1]\n        sub.iloc[i].TopConfigs = s\n    sub.to_csv(\"/kaggle/working/sub.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-07T14:55:17.096306Z","iopub.execute_input":"2023-10-07T14:55:17.096638Z","iopub.status.idle":"2023-10-07T14:55:17.116450Z","shell.execute_reply.started":"2023-10-07T14:55:17.096611Z","shell.execute_reply":"2023-10-07T14:55:17.114904Z"},"trusted":true},"execution_count":null,"outputs":[]}]}