{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-06T17:13:29.725897Z","iopub.execute_input":"2023-09-06T17:13:29.726284Z","iopub.status.idle":"2023-09-06T17:13:32.725034Z","shell.execute_reply.started":"2023-09-06T17:13:29.726225Z","shell.execute_reply":"2023-09-06T17:13:32.72403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/random/train')","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:13:45.750974Z","iopub.execute_input":"2023-09-06T17:13:45.751572Z","iopub.status.idle":"2023-09-06T17:13:45.761276Z","shell.execute_reply.started":"2023-09-06T17:13:45.751526Z","shell.execute_reply":"2023-09-06T17:13:45.76006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the pandas library for data manipulation\nimport pandas as pd\n\n# Import the numpy library for numerical operations\nimport numpy as np\n\n# Import the tqdm library for creating progress bars\nimport tqdm\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:16:49.397409Z","iopub.execute_input":"2023-09-06T17:16:49.397798Z","iopub.status.idle":"2023-09-06T17:16:49.401765Z","shell.execute_reply.started":"2023-09-06T17:16:49.397765Z","shell.execute_reply":"2023-09-06T17:16:49.40104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Define directory paths for different training datasets\nnlp_default_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/default/train'\nnlp_random_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/random/train'\nxla_default_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/default/train'\nxla_random_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/random/train'\n\n# List files in each directory\nnlp_default_train_files = os.listdir(nlp_default_train_dir)\nnlp_random_train_files = os.listdir(nlp_random_train_dir)\nxla_default_train_files = os.listdir(xla_default_train_dir)\nxla_random_train_files = os.listdir(xla_random_train_dir)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:15:19.367358Z","iopub.execute_input":"2023-09-06T17:15:19.367735Z","iopub.status.idle":"2023-09-06T17:15:19.387033Z","shell.execute_reply.started":"2023-09-06T17:15:19.367705Z","shell.execute_reply":"2023-09-06T17:15:19.386231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define directory paths for different training datasets\nnlp_default_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/default/train'\nnlp_random_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/random/train'\nxla_default_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/default/train'\nxla_random_train_dir = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/random/train'\n\n# List files in each directory\nndt_files = os.listdir(nlp_default_train_dir)\nnrt_files = os.listdir(nlp_random_train_dir)\nxdt_files = os.listdir(xla_default_train_dir)\nxrt_files = os.listdir(xla_random_train_dir)\n\n# Load data from files using tqdm for progress bars\nndt = [\n    np.load(os.path.join(nlp_default_train_dir, value))\n    for value in tqdm(ndt_files, total=len(ndt_files), desc='Loading NDT --->')\n]\n\nnrt = [\n    np.load(os.path.join(nlp_random_train_dir, value))\n    for value in tqdm(nrt_files, total=len(nrt_files), desc='Loading NRT --->')\n]\n\nxdt = [\n    np.load(os.path.join(xla_default_train_dir, value))\n    for value in tqdm(xdt_files, total=len(xdt_files), desc='Loading XDT --->')\n]\n\nxrt = [\n    np.load(os.path.join(xla_random_train_dir, value))\n    for value in tqdm(xrt_files, total=len(xrt_files), desc='Loading XRT --->')\n]\n\n# Combine all loaded files into a single list\nfiles = ndt + nrt + xdt + xrt\n\n# Calculate the total number of loaded files\nnum_files = len(files)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:17:39.965244Z","iopub.execute_input":"2023-09-06T17:17:39.965659Z","iopub.status.idle":"2023-09-06T17:17:48.916138Z","shell.execute_reply.started":"2023-09-06T17:17:39.965626Z","shell.execute_reply":"2023-09-06T17:17:48.915016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n\n# Incorrect usage, which leads to the \"module' object is not callable\" error\ntqdm()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:17:35.484262Z","iopub.execute_input":"2023-09-06T17:17:35.485131Z","iopub.status.idle":"2023-09-06T17:17:35.498759Z","shell.execute_reply.started":"2023-09-06T17:17:35.485096Z","shell.execute_reply":"2023-09-06T17:17:35.497242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize empty lists to store data\nnode_feat_list = []\nnode_opcode_list = []\nedge_index_list = []\nnode_config_ids_list = []\nconfig_runtime_list = []\nnode_splits_list = []\n\n# Iterate through files to extract data\nfor val in tqdm(files, total=len(files), desc='Reading Data --->'):\n    node_feat_list.append(val['node_feat'])\n    node_opcode_list.append(val['node_opcode'])\n    edge_index_list.append(val['edge_index'])\n    node_config_ids_list.append(val['node_config_ids'])\n    config_runtime_list.append(val['config_runtime'])\n    node_splits_list.append(val['node_splits'])\n\n# Create a DataFrame from the extracted data\ndata = pd.DataFrame({\n    'node_feat': node_feat_list,\n    'node_opcode': node_opcode_list,\n    'edge_index': edge_index_list,\n    'node_config_ids': node_config_ids_list,\n    'config_runtime': config_runtime_list,\n    'node_splits': node_splits_list\n})\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:19:00.838661Z","iopub.execute_input":"2023-09-06T17:19:00.839085Z","iopub.status.idle":"2023-09-06T17:19:11.855948Z","shell.execute_reply.started":"2023-09-06T17:19:00.839046Z","shell.execute_reply":"2023-09-06T17:19:11.854922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:19:38.512426Z","iopub.execute_input":"2023-09-06T17:19:38.513697Z","iopub.status.idle":"2023-09-06T17:19:39.355868Z","shell.execute_reply.started":"2023-09-06T17:19:38.513651Z","shell.execute_reply":"2023-09-06T17:19:39.354767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.to_csv('/kaggle/working/Input Files.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-06T17:20:01.911914Z","iopub.execute_input":"2023-09-06T17:20:01.912717Z","iopub.status.idle":"2023-09-06T17:20:02.88324Z","shell.execute_reply.started":"2023-09-06T17:20:01.912678Z","shell.execute_reply":"2023-09-06T17:20:02.882354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}