{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"First 3 cells from: <br>\nhttps://www.kaggle.com/code/carlosborrajo/google-fast-or-slow-eda-draft<br>\n\nI am trying to understand what this competition is all about, and be familiar to the data type.<br>\nI am not sure whether I will ever submit for this competition, but will try to understand something :)","metadata":{}},{"cell_type":"code","source":"# Importing the necessary libraries\n\nimport numpy as np\nimport pandas as pd\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-10-18T14:18:19.595841Z","iopub.execute_input":"2023-10-18T14:18:19.596234Z","iopub.status.idle":"2023-10-18T14:18:19.624852Z","shell.execute_reply.started":"2023-10-18T14:18:19.596203Z","shell.execute_reply":"2023-10-18T14:18:19.623875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/predict-ai-model-runtime/sample_submission.csv')\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:30.615271Z","iopub.execute_input":"2023-10-18T05:26:30.615614Z","iopub.status.idle":"2023-10-18T05:26:30.660264Z","shell.execute_reply.started":"2023-10-18T05:26:30.615584Z","shell.execute_reply":"2023-10-18T05:26:30.659195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    if len(filenames) != 0:\n        if filenames[0] != \"sample_submission.csv\":\n            avg = np.array([os.path.getsize(os.path.join(dirname, filename)) for filename in filenames]).mean()\n            print(dirname, len(os.listdir(dirname)))\n            print(\"Size: {:.3f} KB\".format(avg/1024))\n            ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-18T05:26:30.661776Z","iopub.execute_input":"2023-10-18T05:26:30.662083Z","iopub.status.idle":"2023-10-18T05:26:34.689264Z","shell.execute_reply.started":"2023-10-18T05:26:30.662055Z","shell.execute_reply":"2023-10-18T05:26:34.687674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# /kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/random/valid\ntemp = os.listdir(\"/kaggle/input/predict-ai-model-runtime/npz_all/npz\")\ntemp","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-18T05:26:34.692428Z","iopub.execute_input":"2023-10-18T05:26:34.692747Z","iopub.status.idle":"2023-10-18T05:26:34.699877Z","shell.execute_reply.started":"2023-10-18T05:26:34.692719Z","shell.execute_reply":"2023-10-18T05:26:34.698655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layout_nlp_random_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/random/train/albert_en_base_batch_size_16_test.npz'))\nprint(layout_nlp_random_sample.keys())\nprint()\nlayout_nlp_random_test_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/random/test/016ac66a44a906a695afd2228509046a.npz'))\nprint(layout_nlp_random_test_sample.keys())","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:34.701023Z","iopub.execute_input":"2023-10-18T05:26:34.701331Z","iopub.status.idle":"2023-10-18T05:26:36.476906Z","shell.execute_reply.started":"2023-10-18T05:26:34.701303Z","shell.execute_reply":"2023-10-18T05:26:36.475725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layout_nlp_default_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/nlp/default/train/albert_en_base_batch_size_16_train.npz'))\nlayout_nlp_default_sample.keys()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:36.478057Z","iopub.execute_input":"2023-10-18T05:26:36.478425Z","iopub.status.idle":"2023-10-18T05:26:38.342214Z","shell.execute_reply.started":"2023-10-18T05:26:36.478358Z","shell.execute_reply":"2023-10-18T05:26:38.341135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layout_xla_default_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/default/train/alexnet_train_batch_32.npz'))\nlayout_xla_default_sample.keys()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:38.343667Z","iopub.execute_input":"2023-10-18T05:26:38.344513Z","iopub.status.idle":"2023-10-18T05:26:38.545741Z","shell.execute_reply.started":"2023-10-18T05:26:38.344473Z","shell.execute_reply":"2023-10-18T05:26:38.544309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layout_xla_default_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/layout/xla/default/train/alexnet_train_batch_32.npz'))\nlayout_xla_default_sample.keys()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:38.547349Z","iopub.execute_input":"2023-10-18T05:26:38.548046Z","iopub.status.idle":"2023-10-18T05:26:38.731262Z","shell.execute_reply.started":"2023-10-18T05:26:38.548012Z","shell.execute_reply":"2023-10-18T05:26:38.730457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tile_xla_sample = dict(np.load('/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/train/alexnet_train_batch_32_-1bae27a41d70f4dc.npz'))\ntile_xla_sample.keys()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:26:38.732616Z","iopub.execute_input":"2023-10-18T05:26:38.732928Z","iopub.status.idle":"2023-10-18T05:26:38.745108Z","shell.execute_reply.started":"2023-10-18T05:26:38.732900Z","shell.execute_reply":"2023-10-18T05:26:38.743816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in temp: # layout & tile\n    if i == 'layout':\n        \n        temp_ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}')\n        for j in temp_: # nlp & xla\n            temp__ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/{j}')\n            for k in temp__: # default & random\n                temp___ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/{j}/{k}')\n                for l in temp___: # test& train & valid\n                    temp____ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/{j}/{k}/{l}')\n                    count = 0\n                    for npz_file in temp____: # each file\n                        d = dict(np.load(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/{j}/{k}/{l}/{npz_file}'))\n                        print(f\"{i}/{j}/{npz_file} number of keys : {len(d.keys())}\\n\")\n                        print(f\"{i}/{j}/{npz_file} number of values : {len(d.values())}\\n\")\n                        for n in range(len(d.values())):\n                            print(f\"{i}/{j}/{npz_file} number of keys of values\\n{'vvvvvvvvvvvvvvvvvvvvvvv'}\\n{len(list(d.keys())[n])}\")\n                            print(f\"{i}/{j}/{npz_file} number of values of values\\n{'vvvvvvvvvvvvvvvvvvvvvvv'}\\n{len(list(d.values())[n])}\")\n                        print()\n                        print('-------------------------------------------------------')\n                        print()\n                        count += 1\n                        if count == 5:\n                            break\n                    print('==================================================')\n            print()\n\n    if i == 'tile':\n        temp_ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/xla')\n        for j in temp_: # test & train & valid\n            temp__ = os.listdir(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/xla/{j}')\n            count = 0\n            for npz_file in temp__: # each file\n                d = dict(np.load(f'/kaggle/input/predict-ai-model-runtime/npz_all/npz/{i}/xla/{j}/{npz_file}'))\n                print(f\"{i}/{npz_file} keys : {len(d.keys())}\\n\")\n                print(f\"{i}/{npz_file} number of values : {len(d.values())}\\n\")\n                for n in range(len(d.values())):\n                    print(f\"{i}/{npz_file} number of keys of values\\n{'vvvvvvvvvvvvvvvvvvvvvvv'}\\n{len(list(d.keys())[n])}\")\n                    print(f\"{i}/{npz_file} number of values of values\\n{'vvvvvvvvvvvvvvvvvvvvvvv'}\\n{len(list(d.values())[n])}\")\n                print()\n                print('-------------------------------------------------------')\n                print()\n                count += 1\n                if count == 2:\n                    break\n            print('==================================================')\n            print()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T05:28:56.469117Z","iopub.execute_input":"2023-10-18T05:28:56.469573Z","iopub.status.idle":"2023-10-18T05:29:38.037656Z","shell.execute_reply.started":"2023-10-18T05:28:56.469536Z","shell.execute_reply":"2023-10-18T05:29:38.036484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}