{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":9039692,"sourceType":"datasetVersion","datasetId":5449602},{"sourceId":9039962,"sourceType":"datasetVersion","datasetId":5449505},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the code that reproduces a Private Score of 0.94908.\n\n- \"Magic\" is not used. The only token-saving method used is the AND omission shared at https://www.kaggle.com/competitions/uspto-explainable-ai/discussion/516104.\n- Some refactoring has been done, but the core part remains unchanged.\n- The solver includes a time-cutoff process, so the Private Score does not match exactly.\n\nFor more details: https://www.kaggle.com/competitions/uspto-explainable-ai/discussion/522202","metadata":{}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"# import whoosh_utils\nimport os\nimport polars as pl\n\nLOCAL = not os.path.exists(\"/kaggle\")\nDATADIR = \"/kaggle/input/uspto-explainable-ai\"\nMY_DATADIR = \"/kaggle/input/uspto2024dataset2\"\nMY_SCRIPTDIR = \"/kaggle/input/uspto2024script2\"\nOUTDIR = \"/kaggle/working\"\nTESTFILE = f\"{DATADIR}/test.csv\"\n# TESTFILE = f\"{MY_DATADIR}/test.csv\"  # used for local validation\nif LOCAL:\n    DATADIR = \"/workspaces/uspto/dataset\"\n    MY_DATADIR = \"/workspaces/uspto/dataset/me\"\n    MY_SCRIPTDIR = \"/workspaces/uspto/src\"\n    OUTDIR = \"/workspaces/uspto/temporary\"\n    TESTFILE = f\"{DATADIR}/test_2500/1.csv\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-31T15:39:24.282458Z","iopub.execute_input":"2024-08-31T15:39:24.283069Z","iopub.status.idle":"2024-08-31T15:39:24.557720Z","shell.execute_reply.started":"2024-08-31T15:39:24.283035Z","shell.execute_reply":"2024-08-31T15:39:24.556682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"test = pl.read_csv(TESTFILE)\nif len(test) == 10:\n    test = pl.read_csv(f\"{MY_DATADIR}/test.csv\")\ndisplay(test)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:14.170325Z","iopub.execute_input":"2024-07-26T12:48:14.171138Z","iopub.status.idle":"2024-07-26T12:48:14.343669Z","shell.execute_reply.started":"2024-07-26T12:48:14.171101Z","shell.execute_reply":"2024-07-26T12:48:14.342631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pubnum_list = []\nquery_pubnum_list = []\nfor row in test.iter_rows():\n    query_pubnum_list.append(row[0])\n    pubnum_list.extend(row[1:])\ntest_pubnums = pl.DataFrame({\"publication_number\": pubnum_list}).unique(maintain_order=True)\nquery_pubnums = pl.DataFrame({\"publication_number\": query_pubnum_list}).unique(maintain_order=True)\nquery_pubnums = query_pubnums.filter(~pl.col(\"publication_number\").is_in(test_pubnums[\"publication_number\"]))\nprint(test_pubnums.shape)\nprint(query_pubnums.shape)\ndisplay(test_pubnums)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:14.345254Z","iopub.execute_input":"2024-07-26T12:48:14.345953Z","iopub.status.idle":"2024-07-26T12:48:14.418947Z","shell.execute_reply.started":"2024-07-26T12:48:14.345915Z","shell.execute_reply":"2024-07-26T12:48:14.417934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pl.scan_parquet(f\"{DATADIR}/patent_metadata.parquet\")\nmetadata = metadata.select([\"publication_number\",\"cpc_codes\"]).collect()\n# potential_negative = metadata.select([\"publication_number\"]).sample(1000, seed=42)\n# potential_negative = metadata.select([\"publication_number\"]).sample(2000000, seed=42)\npotential_negative = metadata.select([\"publication_number\"])\npotential_negative = potential_negative.filter(~pl.col(\"publication_number\").is_in(test_pubnums[\"publication_number\"]))\npotential_negative = potential_negative.filter(~pl.col(\"publication_number\").is_in(query_pubnums[\"publication_number\"]))\npubnums = pl.concat([test_pubnums.to_series(), query_pubnums.to_series(), potential_negative.to_series()])\nn_test = test_pubnums.shape[0]\nn_potential_negative = potential_negative.shape[0]\nn_all = pubnums.shape[0]\nprint(n_test, n_potential_negative, n_all)\npubnums = pubnums.to_frame() # series -> dataframe\npubnums = pubnums.with_row_index(\"id\")\nprint(pubnums.head())\npubnum2id = {}\nfor row in pubnums.rows():\n    pubnum2id[row[1]] = row[0]\npubnums = pubnums.select([\"publication_number\"])\ndisplay(pubnums)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:14.420039Z","iopub.execute_input":"2024-07-26T12:48:14.420363Z","iopub.status.idle":"2024-07-26T12:48:31.745240Z","shell.execute_reply.started":"2024-07-26T12:48:14.420336Z","shell.execute_reply":"2024-07-26T12:48:31.744227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pubnums_id = test_pubnums.with_row_index(\"id\")\ntest_pubnum2id = {}\nfor row in test_pubnums_id.rows():\n    test_pubnum2id[row[1]] = row[0]\ntest_pubnums_id = test_pubnums_id.select([\"publication_number\", \"id\"])\ntest_pubnums_id.write_csv(f\"{OUTDIR}/vocab_pubnum.tsv\", separator='\\t', include_header=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:31.748053Z","iopub.execute_input":"2024-07-26T12:48:31.748387Z","iopub.status.idle":"2024-07-26T12:48:31.757202Z","shell.execute_reply.started":"2024-07-26T12:48:31.748359Z","shell.execute_reply":"2024-07-26T12:48:31.756252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_id = test.select(\n    [pl.col(col).map_elements(lambda x: pubnum2id.get(x, -1), return_dtype=pl.Int64).alias(col) for col in test.columns]\n)\ntest_id.write_csv(f\"{OUTDIR}/test.tsv\", separator='\\t', include_header=False)\ndisplay(test_id)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:31.758511Z","iopub.execute_input":"2024-07-26T12:48:31.758808Z","iopub.status.idle":"2024-07-26T12:48:31.817240Z","shell.execute_reply.started":"2024-07-26T12:48:31.758782Z","shell.execute_reply":"2024-07-26T12:48:31.816238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nshot = pl.read_csv(f\"{MY_DATADIR}/nshot.tsv\", separator=\"\\t\")\ntest_nshot = test_pubnums.join(nshot, how=\"inner\", on=\"publication_number\")\nprint(len(test_nshot))\ntest_nshot.head()\n\ndef map_pubnum_to_id(pubnum):\n    return pubnum2id.get(pubnum, None)\n\ntest_nshot = test_nshot.with_columns(\n    pl.col(\"publication_number\").map_elements(map_pubnum_to_id, return_dtype=pl.Int32).alias(\"pubid\")\n)\ntest_nshot = test_nshot.select([\"pubid\", \"tokens\"])\ntest_nshot.write_csv(f\"{OUTDIR}/nshot.tsv\", separator=\"\\t\", include_header=False)\ndisplay(test_nshot)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:31.818351Z","iopub.execute_input":"2024-07-26T12:48:31.818653Z","iopub.status.idle":"2024-07-26T12:48:34.505838Z","shell.execute_reply.started":"2024-07-26T12:48:31.818625Z","shell.execute_reply":"2024-07-26T12:48:34.504862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_and_write_vocab(df, key, write_key=None):\n    if write_key is None:\n        write_key = key\n    words = set()\n    for row in df.select(key).iter_rows():\n        for word in row[0]:\n            words.add(word)\n    words = list(words)\n    words.sort()\n    words = pl.DataFrame({key: words})\n    words = words.with_row_index(\"id\")\n    words = words.select([key, \"id\"])\n    print(len(words))\n    words.write_csv(f\"{OUTDIR}/vocab_{write_key}.tsv\", separator='\\t', include_header=False)\n    word2id = {}\n    for row in words.rows():\n        word2id[row[0]] = row[1]\n    return word2id\n\ndef write_x2y(filepath: str, x2y):\n    with open(filepath, \"w\") as f:\n        f.write(f\"{len(x2y)}\\n\")\n        for y in x2y:\n            f.write(f\"{len(y)}\")\n            if len(y) == 0:\n                f.write(\"\\n\")\n                continue\n            ystr = \" \".join(map(str, y))\n            f.write(f\" {ystr}\\n\")\n\ndef calc_y2xs(df, id_x, id_y, vocab_x, vocab_y, write_key_y):\n    y2xs = [[] for _ in range(len(vocab_y))]\n    for row in df.iter_rows():\n        x = row[id_x]\n        xid = vocab_x[x]\n        for y in row[id_y]:\n            yid = vocab_y.get(y)\n            if yid is None:\n                continue\n            y2xs[yid].append(xid)\n    file = f\"{OUTDIR}/{write_key_y}2pubnum.txt\"\n    write_x2y(file, y2xs)\n    print(file)\n\ndef calc_x2ys(df, id_x, id_y, vocab_x, vocab_y, write_key_y):\n    x2ys = [[] for _ in range(len(vocab_x))]\n    for row in df.iter_rows():\n        x = row[id_x]\n        xid = vocab_x[x]\n        x2ys[xid] = [vocab_y[y] for y in row[id_y]]\n    file = f\"{OUTDIR}/pubnum2{write_key_y}.txt\"\n    write_x2y(file, x2ys)  \n    print(file)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:34.507258Z","iopub.execute_input":"2024-07-26T12:48:34.507561Z","iopub.status.idle":"2024-07-26T12:48:34.519935Z","shell.execute_reply.started":"2024-07-26T12:48:34.507535Z","shell.execute_reply":"2024-07-26T12:48:34.518881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pubnums.select(\"publication_number\")\ndf = df.join(metadata, on=\"publication_number\", how=\"left\")\ndisplay(df)\ndf_test = df.join(test_pubnums, on=\"publication_number\", how=\"inner\")\ncpc2id = create_and_write_vocab(df_test, \"cpc_codes\", \"cpc\")\n\nID_PUBNUM = 0\nID_CPC = 1\n\ncalc_x2ys(df_test, ID_PUBNUM, ID_CPC, test_pubnum2id, cpc2id, \"cpc\")\ncalc_y2xs(df, ID_PUBNUM, ID_CPC, pubnum2id, cpc2id, \"cpc\")","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:48:34.521394Z","iopub.execute_input":"2024-07-26T12:48:34.521832Z","iopub.status.idle":"2024-07-26T12:49:14.891614Z","shell.execute_reply.started":"2024-07-26T12:48:34.521795Z","shell.execute_reply":"2024-07-26T12:49:14.890536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel metadata\ndel df, df_test\ndel cpc2id\ndel nshot, test_nshot\ndel potential_negative\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-26T12:49:14.893345Z","iopub.execute_input":"2024-07-26T12:49:14.893790Z","iopub.status.idle":"2024-07-26T12:49:15.373676Z","shell.execute_reply.started":"2024-07-26T12:49:14.893754Z","shell.execute_reply":"2024-07-26T12:49:15.372059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor field_info in [(\"description\", \"10k\"), (\"title\", \"400k\"), (\"abstract\", \"400k\"), (\"claims\", \"100k\")]:\n    field, freq = field_info\n    df_x = pl.scan_parquet(f\"{MY_DATADIR}/{field}_{freq}.parquet\").collect()\n    print(field, len(df_x))\n    df_x_test = df_x.join(test_pubnums, on=\"publication_number\", how=\"inner\")\n    print(field, len(df_x_test))\n    display(df_x)\n    x2id = create_and_write_vocab(df_x_test, field)\n    ID_PUBNUM = 0\n    ID_X = 1\n    calc_x2ys(df_x_test, ID_PUBNUM, ID_X, test_pubnum2id, x2id, field)\n    calc_y2xs(df_x, ID_PUBNUM, ID_X, pubnum2id, x2id, field)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T13:01:58.497954Z","iopub.execute_input":"2024-07-26T13:01:58.500203Z","iopub.status.idle":"2024-07-26T13:13:12.320986Z","shell.execute_reply.started":"2024-07-26T13:01:58.500123Z","shell.execute_reply":"2024-07-26T13:13:12.319213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel df_x, df_x_test\ndel pubnum2id, x2id\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-26T13:13:12.324132Z","iopub.execute_input":"2024-07-26T13:13:12.324637Z","iopub.status.idle":"2024-07-26T13:13:13.643822Z","shell.execute_reply.started":"2024-07-26T13:13:12.324599Z","shell.execute_reply":"2024-07-26T13:13:13.642819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Solve and Validate","metadata":{}},{"cell_type":"code","source":"%%time\nimport subprocess\n\n## Execution on 2500 lines takes approximately 4-6 hours.\nresult = subprocess.run(f\"g++ -std=c++17 -O3 {MY_SCRIPTDIR}/main.cpp && ./a.out --field cpc --field title --field abstract --field claims --field description\", shell=True, text=True, capture_output=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-26T13:18:12.438061Z","iopub.execute_input":"2024-07-26T13:18:12.438577Z","iopub.status.idle":"2024-07-26T13:21:28.098509Z","shell.execute_reply.started":"2024-07-26T13:18:12.438538Z","shell.execute_reply":"2024-07-26T13:21:28.096985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"queries = result.stdout.strip().split(\"\\n\")\nprint(result.stderr)\nassert len(queries) == len(test)\nwith open(\"submission.csv\", \"w\") as f:\n    f.write(\"publication_number,query\\n\")\n    for i in range(len(queries)):\n        pub = test[\"publication_number\"][i]\n        f.write(f\"{pub},{queries[i]}\\n\")\n!head -n 11 submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-07-26T13:21:28.100866Z","iopub.execute_input":"2024-07-26T13:21:28.101821Z","iopub.status.idle":"2024-07-26T13:21:29.313916Z","shell.execute_reply.started":"2024-07-26T13:21:28.101781Z","shell.execute_reply":"2024-07-26T13:21:29.312544Z"},"trusted":true},"execution_count":null,"outputs":[]}]}