{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":204479873,"sourceType":"kernelVersion"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Happy new year guys! \n\nThis is a simulation API for the submit.\nIt is really important to understand what happens during the scoring stage!\n\nWhen will our predict function be called? When will we get the lags?\n\nAlthough these stuff are already introduced in the document, still, I got lot of \"NoteBook Threw Exception\" in my submission.\nAnyway, I am a new kaggler and this happend when I was trying to introduce the online learning blocks to my model.\n\nI can get nothing more than \"NoteBook Threw Exception\".\n\nFortunately, this simulation API helps me to debug my code.\n\nIf you are facing the same problem(run well in local notebook but \"NoteBook Threw Exception\" when submit) \nJust ctrl+v this block to your code and run, you will get more detail for your bugs.\n\nHope this can help some guys like me!","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport re\nfrom glob import glob\nfrom tqdm import tqdm  \n# 获取所有 date_id 的文件路径\ntest_files = sorted(glob(os.path.join(\"/kaggle/input/janestreet-updated-simulator-for-time-series-api/debug/test.parquet/\", \"date_id=*\", \"part-0.parquet\")))\nlags_files = sorted(glob(os.path.join(\"/kaggle/input/janestreet-updated-simulator-for-time-series-api/debug/lags.parquet/\", \"date_id=*\", \"part-0.parquet\")))\ntest_files = sorted(test_files, key=lambda x: int(re.search(r\"date_id=(\\d+)\", x).group(1)))\nlags_files = sorted(lags_files, key=lambda x: int(re.search(r\"date_id=(\\d+)\", x).group(1)))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndate_id_pattern = re.compile(r\"date_id=(\\d+)\")\n\n# list for saving the prediction\nresults = []\n\nprev_date_id = None  # Record for last date_id\n\nfor test_file in tqdm(test_files, desc=\"Processing Test Files\"):\n    # get current_date_id\n    match = date_id_pattern.search(test_file)\n    if not match:\n        continue\n    current_date_id = int(match.group(1))\n\n    \n    test_data = pl.read_parquet(test_file)\n\n    # the predict function will be called on different time_id(time_id=0,1,2...)\n    for time_id in tqdm(test_data[\"time_id\"].unique(), desc=f\"Processing date_id={current_date_id}\", leave=False):\n        time_batch = test_data.filter(pl.col(\"time_id\") == time_id)\n\n        if current_date_id != prev_date_id and time_id == 0:\n            # ①If we go to the next date_id, we can get the full lags data when time_id==0.\n            lags_file = next((f for f in lags_files if f\"date_id={current_date_id}\" in f), None)\n            lags_data = pl.read_parquet(lags_file) if lags_file else None\n            print(f\"Processing new date_id: {current_date_id}, time_id: {time_id} with lags\")\n        else:\n            # ②And we get None for the rest of the time_id(1,2,3...)\n            lags_data = None\n            print(f\"Processing date_id: {current_date_id}, time_id: {time_id} without lags\")\n\n        # call predict func for every time_id batch\n        prediction = predict(time_batch, lags_data)\n        results.append(prediction)\n\n    # Update prev_date_id\n    prev_date_id = current_date_id\n\nresults_df = pl.concat(results)\nprint(\"Prediction summary:\")\nprint(results_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}