{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Code along video: [https://youtu.be/53P7u6IDZnc](https://youtu.be/53P7u6IDZnc)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import LinearRegression\nimport os","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IDX = [6, 7, 14, 15]\n\ndef load_data(files):\n    X, y = [], []\n\n    for filename in files:\n        unnormalised_runtime = np.load(filename, allow_pickle=True)['config_runtime']\n        runtime_normalisers = np.load(filename, allow_pickle=True)['config_runtime_normalizers']\n        runtimes = unnormalised_runtime / runtime_normalisers\n\n        output_bounds_sums = np.load(filename, allow_pickle=True)['config_feat'][:, IDX]\n\n        X.extend(output_bounds_sums)\n        y.extend(runtimes)\n\n    return np.array(X), np.array(y).reshape(-1, 1)\n\n\ndef slowdown(y_true, y_pred, k):\n    y_fastest = np.min(y_true)\n\n    top_k_indices = y_pred[:k]\n\n    top_k_predicted_times = np.min(y_true[top_k_indices])\n\n    ratio = top_k_predicted_times / y_fastest\n\n    return ratio - 1\n\ndef speed_score(y_true, y_pred, k):\n    return 1 - slowdown(y_true, y_pred, k)\n\n\nBASE_DIR = '/kaggle/input/predict-ai-model-runtime/npz_all/npz/tile/xla/'\nTRAIN_DIR = BASE_DIR + 'train/'\nVALID_DIR = BASE_DIR + 'valid/'\nTEST_DIR = BASE_DIR + 'test/'","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:21:54.094235Z","iopub.execute_input":"2023-11-09T05:21:54.094684Z","iopub.status.idle":"2023-11-09T05:21:54.107523Z","shell.execute_reply.started":"2023-11-09T05:21:54.094631Z","shell.execute_reply":"2023-11-09T05:21:54.106101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\ntrain_files = [TRAIN_DIR + f for f in os.listdir(TRAIN_DIR)]\nvalid_files = [VALID_DIR + f for f in os.listdir(VALID_DIR)]\n\nX, y = load_data(train_files)\n\nprint('X shape', X.shape)\nprint('y shape', y.shape)\n\nmodel = LinearRegression()\nmodel.fit(X, y)\nprint(\"Model coefficients:\", model.coef_)\nprint(\"Model intercept:\", model.intercept_)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:22:18.180502Z","iopub.execute_input":"2023-11-09T05:22:18.181377Z","iopub.status.idle":"2023-11-09T05:23:16.283574Z","shell.execute_reply.started":"2023-11-09T05:22:18.181331Z","shell.execute_reply":"2023-11-09T05:23:16.282735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare linear regression vs uniform mean prediction for validation set\ntrain_mse = mean_squared_error(y, model.predict(X))\nprint(\"MSE on train set:\", train_mse)\n\nX_val, y_val = load_data(valid_files)\ny_pred = model.predict(X_val)\nmse = mean_squared_error(y_val, y_pred)\nprint(\"Validation Mean Squared Error:\", mse)\n\nmean_baseline_mse = mean_squared_error(y_val, np.mean(y_val) * np.ones_like(y_val))\nprint(\"Validation MSE for mean prediction\", mean_baseline_mse)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:23:45.519781Z","iopub.execute_input":"2023-11-09T05:23:45.520226Z","iopub.status.idle":"2023-11-09T05:23:51.888035Z","shell.execute_reply.started":"2023-11-09T05:23:45.520190Z","shell.execute_reply":"2023-11-09T05:23:51.886231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare linear regression vs random for predicting runtime order\n\nperfect_scores = []\nrand_scores = []\nlinear_scores = []\n\nfor filename in valid_files:\n    unnormalised_runtime = np.load(filename, allow_pickle=True)['config_runtime']\n    runtime_normalisers = np.load(filename, allow_pickle=True)['config_runtime_normalizers']\n    runtimes = unnormalised_runtime / runtime_normalisers\n\n    perfect_preds = np.argsort(runtimes)\n    perfect_score = speed_score(runtimes, perfect_preds, 3)\n    perfect_scores.append(perfect_score)\n    \n    # randomize the indices\n    np.random.shuffle(perfect_preds)\n    rand_score = speed_score(runtimes, perfect_preds, 3)\n    rand_scores.append(rand_score)\n    \n    predicted_runtimes = model.predict(np.load(filename, allow_pickle=True)['config_feat'][:, IDX])\n    predicted_idx = np.argsort(predicted_runtimes[:, 0])\n    linear_score = speed_score(runtimes, predicted_idx, 3)\n    linear_scores.append(linear_score)\n\nprint(\"perfect score\", np.mean(perfect_scores))\nprint(\"random score\", np.mean(rand_scores))\nprint(\"linear score\", np.mean(linear_scores))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:24:44.113304Z","iopub.execute_input":"2023-11-09T05:24:44.114302Z","iopub.status.idle":"2023-11-09T05:24:48.902869Z","shell.execute_reply.started":"2023-11-09T05:24:44.114261Z","shell.execute_reply":"2023-11-09T05:24:48.901634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '/kaggle/input/predict-ai-model-runtime/sample_submission.csv'\ndf = pd.read_csv(sample_submission_path)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:26:23.905408Z","iopub.execute_input":"2023-11-09T05:26:23.905892Z","iopub.status.idle":"2023-11-09T05:26:23.946538Z","shell.execute_reply.started":"2023-11-09T05:26:23.905855Z","shell.execute_reply":"2023-11-09T05:26:23.945309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = load_data(train_files)\nX2, y2 = load_data(valid_files)\n\nX = np.concatenate([X, X2], axis=0)\nprint(X.shape)\ny = np.concatenate([y, y2], axis=0)\n\nmodel = LinearRegression()\nmodel.fit(X, y)\n\nprint(\"Full Tile Model coefficients:\", model.coef_)\nprint(\"Full Tile Model intercept:\", model.intercept_)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:26:49.132871Z","iopub.execute_input":"2023-11-09T05:26:49.133324Z","iopub.status.idle":"2023-11-09T05:27:29.835787Z","shell.execute_reply.started":"2023-11-09T05:26:49.133288Z","shell.execute_reply":"2023-11-09T05:27:29.834617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_linear_ordering(identifier, model):\n    if \"tile:xla:\" in identifier:\n        filename = TEST_DIR + identifier.split(\":\")[-1] + '.npz'\n        X = np.load(filename, allow_pickle=True)['config_feat'][:, IDX]\n        y_pred = model.predict(X)\n        \n        indices = np.argsort(y_pred[:, 0])\n        \n        return \";\".join([str(i) for i in list(indices[:5])])    \n    else:\n        return \";\".join(map(str, np.random.choice(range(10), 5, replace=False)))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:27:29.837854Z","iopub.execute_input":"2023-11-09T05:27:29.838214Z","iopub.status.idle":"2023-11-09T05:27:29.845369Z","shell.execute_reply.started":"2023-11-09T05:27:29.838183Z","shell.execute_reply":"2023-11-09T05:27:29.844541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nbaseline_submission = pd.DataFrame({\n    'ID': df['ID'],\n    'TopConfigs': [generate_linear_ordering(i, model) for i in df['ID'].values]\n})\n\nbaseline_submission","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:27:29.846473Z","iopub.execute_input":"2023-11-09T05:27:29.847178Z","iopub.status.idle":"2023-11-09T05:27:37.795368Z","shell.execute_reply.started":"2023-11-09T05:27:29.847145Z","shell.execute_reply":"2023-11-09T05:27:37.794190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T05:27:37.797129Z","iopub.execute_input":"2023-11-09T05:27:37.797433Z","iopub.status.idle":"2023-11-09T05:27:37.810198Z","shell.execute_reply.started":"2023-11-09T05:27:37.797406Z","shell.execute_reply":"2023-11-09T05:27:37.809126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}