{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport pyarrow.dataset as ds\nimport pyarrow as pa\nimport os\nimport json","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:13.240017Z","iopub.execute_input":"2024-07-09T08:08:13.240517Z","iopub.status.idle":"2024-07-09T08:08:14.894670Z","shell.execute_reply.started":"2024-07-09T08:08:13.240467Z","shell.execute_reply":"2024-07-09T08:08:14.893001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '/kaggle/input/uspto-explainable-ai'","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:14.897737Z","iopub.execute_input":"2024-07-09T08:08:14.898368Z","iopub.status.idle":"2024-07-09T08:08:14.904015Z","shell.execute_reply.started":"2024-07-09T08:08:14.898334Z","shell.execute_reply":"2024-07-09T08:08:14.902623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_dir = os.path.join(root_dir, 'patent_data')\n\npartitioning = ds.FilenamePartitioning(pa.schema([('year', pa.int16()), ('month', pa.int8())]))\n\nvalid_partitions = []\nfor f in os.listdir(dataset_dir):\n    if f == 'nan_nan.parquet':\n        continue\n    partitioning.parse(f)\n    valid_partitions.append(f)\n    \nprint(len(valid_partitions))\n\ndataset = ds.dataset([os.path.join(dataset_dir, f) for f in valid_partitions], format='parquet')\nlf = pl.scan_pyarrow_dataset(dataset)\nprint(lf.select(pl.len()).collect())\n\nlf.first().collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:14.905497Z","iopub.execute_input":"2024-07-09T08:08:14.905838Z","iopub.status.idle":"2024-07-09T08:08:23.610992Z","shell.execute_reply.started":"2024-07-09T08:08:14.905810Z","shell.execute_reply":"2024-07-09T08:08:23.609095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = json.load(open(os.path.join(root_dir, 'train_index_patent_ids.json')))\n\nids_lf = pl.LazyFrame({'publication_number': train_ids})","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:23.614973Z","iopub.execute_input":"2024-07-09T08:08:23.615495Z","iopub.status.idle":"2024-07-09T08:08:23.814979Z","shell.execute_reply.started":"2024-07-09T08:08:23.615449Z","shell.execute_reply":"2024-07-09T08:08:23.813512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nns_lf = pl.scan_csv(os.path.join(root_dir, 'nearest_neighbors.csv'))\n\nnns_lf.first().collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:23.818316Z","iopub.execute_input":"2024-07-09T08:08:23.819073Z","iopub.status.idle":"2024-07-09T08:08:23.881758Z","shell.execute_reply.started":"2024-07-09T08:08:23.819028Z","shell.execute_reply":"2024-07-09T08:08:23.880365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nns_df = nns_lf.join(ids_lf, on='publication_number', how='inner').collect()\n\nnns_df","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:08:23.883710Z","iopub.execute_input":"2024-07-09T08:08:23.884395Z","iopub.status.idle":"2024-07-09T08:09:36.202572Z","shell.execute_reply.started":"2024-07-09T08:08:23.884359Z","shell.execute_reply":"2024-07-09T08:09:36.201096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_nns = nns_df.unique()\n\nunique_nns","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:36.204922Z","iopub.execute_input":"2024-07-09T08:09:36.205360Z","iopub.status.idle":"2024-07-09T08:09:36.531139Z","shell.execute_reply.started":"2024-07-09T08:09:36.205329Z","shell.execute_reply":"2024-07-09T08:09:36.529910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_nns = ids_lf.join(unique_nns.lazy(), on='publication_number', how='anti').collect()\n\nmissing_nns","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:36.533012Z","iopub.execute_input":"2024-07-09T08:09:36.533469Z","iopub.status.idle":"2024-07-09T08:09:36.568289Z","shell.execute_reply.started":"2024-07-09T08:09:36.533430Z","shell.execute_reply":"2024-07-09T08:09:36.567116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pl.read_parquet(os.path.join(root_dir, 'patent_metadata.parquet'))\n\ntrain_metadata = metadata.join(unique_nns, on='publication_number', how='inner')\n\ntrain_metadata","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:36.569696Z","iopub.execute_input":"2024-07-09T08:09:36.570013Z","iopub.status.idle":"2024-07-09T08:09:45.760745Z","shell.execute_reply.started":"2024-07-09T08:09:36.569987Z","shell.execute_reply":"2024-07-09T08:09:45.758958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate filename from publication date in the same format as the partition scheme\ntrain_metadata = train_metadata.with_columns(\n    pl.format(\"{}_{}.parquet\", pl.col('publication_date').dt.year(), pl.col('publication_date').dt.month()).alias(\"filename\")\n)\n\ntrain_metadata","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:45.765349Z","iopub.execute_input":"2024-07-09T08:09:45.766604Z","iopub.status.idle":"2024-07-09T08:09:45.844158Z","shell.execute_reply.started":"2024-07-09T08:09:45.766555Z","shell.execute_reply":"2024-07-09T08:09:45.842512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"partitions_df = pl.DataFrame({'filename': valid_partitions})\n\npartitions_df","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:45.846078Z","iopub.execute_input":"2024-07-09T08:09:45.846737Z","iopub.status.idle":"2024-07-09T08:09:45.860407Z","shell.execute_reply.started":"2024-07-09T08:09:45.846688Z","shell.execute_reply":"2024-07-09T08:09:45.858682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check that the filenames generated from publication date all exist in the data directory\nverify_train_metadata = train_metadata.join(partitions_df, on='filename', how='inner')\n\nverify_train_metadata","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:45.862144Z","iopub.execute_input":"2024-07-09T08:09:45.862610Z","iopub.status.idle":"2024-07-09T08:09:46.063434Z","shell.execute_reply.started":"2024-07-09T08:09:45.862578Z","shell.execute_reply":"2024-07-09T08:09:46.062276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_data = pl.DataFrame()\n\nfor filename, data in train_metadata.group_by(['filename']):\n    raw_data = pl.read_parquet(os.path.join(dataset_dir, filename[0]))\n    existing = data.join(raw_data, on='publication_number', how='inner')\n    if data.shape[0] != existing.shape[0]:\n        print(f'Raw data not fully available in {filename}')\n        print(data)\n    if full_train_data.is_empty():\n        full_train_data = existing\n    else:\n        full_train_data = pl.concat([full_train_data, existing])\n    print(full_train_data.shape)\n    \nfull_train_data","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:09:46.065799Z","iopub.execute_input":"2024-07-09T08:09:46.066396Z","iopub.status.idle":"2024-07-09T08:51:10.308324Z","shell.execute_reply.started":"2024-07-09T08:09:46.066343Z","shell.execute_reply":"2024-07-09T08:51:10.306052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_data.write_parquet('full_train_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-09T08:51:10.311703Z","iopub.execute_input":"2024-07-09T08:51:10.312158Z","iopub.status.idle":"2024-07-09T08:53:10.653089Z","shell.execute_reply.started":"2024-07-09T08:51:10.312126Z","shell.execute_reply":"2024-07-09T08:53:10.651632Z"},"trusted":true},"execution_count":null,"outputs":[]}]}