{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":187159588,"sourceType":"kernelVersion"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# code requirements  \nPlease execute in the following environment.\n\n* RAM: 64GB\n* kaggle Docker\n","metadata":{}},{"cell_type":"code","source":"\nfrom pathlib import Path\n\nimport polars as pl\nfrom tqdm import tqdm\n\nimport sys\nimport whoosh_utils\n\nIS_TRAIN = True\n\n# 乱数固定\nimport os\n# import torch\nimport numpy as np\nimport random\ndef set_seed(seed=None, cudnn_deterministic=True):\n    if seed is None:\n        seed = 42\n\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    # torch.manual_seed(seed)\n    # torch.cuda.manual_seed(seed)\n    # torch.backends.cudnn.deterministic = cudnn_deterministic  # A100,effnetだとFalseの方が早い\n    # torch.backends.cudnn.benchmark = False\nset_seed()\n    \ncomp_data_dir = Path(\"/kaggle/input/uspto-explainable-ai\")\n\n# Read patent since 1975\nmeta = pl.scan_parquet(comp_data_dir / \"patent_metadata.parquet\")\nmeta = (\n    meta.with_columns(\n        pl.col(\"publication_date\").dt.year().alias(\"year\"),\n        pl.col(\"publication_date\").dt.month().alias(\"month\"),\n    )\n    .filter(pl.col(\"publication_date\") >= pl.date(1975, 1, 1))\n    .rename({\"cpc_codes\": \"cpc\"})\n    .collect()\n)\n\n\n\nif IS_TRAIN:\n    # test_nn = pl.scan_csv(comp_data_dir / \"test.csv\")\n    # test_nn = pl.read_csv(comp_data_dir / \"nearest_neighbors.csv\", n_rows=4000).head(2500).lazy()\n    # test_nn = pl.read_csv(comp_data_dir / \"nearest_neighbors.csv\", n_rows=4000).head(2500)\n    test_nn = pl.read_csv('/kaggle/input/create-valid-index-add-claim/nn_df_for_index.csv')\n    \n    test_nn_pub = test_nn.melt().get_column(\"value\").unique()\n    # 候補数を合わせるためにnegative sampleを抽出 候補数: (2500 + 1500) * 50 = 200,000\n    neg_sample = (\n        pl.read_csv(comp_data_dir / \"nearest_neighbors.csv\")\n        .filter(~pl.col(\"publication_number\").is_in(test_nn_pub))\n        .sample(1500, seed=42)\n    )\n    test_nn = pl.concat([test_nn, neg_sample]).lazy()\nelse:\n    test_nn = pl.read_csv(comp_data_dir / \"test.csv\")\n    # test_nn = pl.read_csv(comp_data_dir / \"nearest_neighbors.csv\", n_rows=4000).head(2500).lazy()\n    \n    test_nn_pub = test_nn.melt().get_column(\"value\").unique()\n    # 候補数を合わせるためにnegative sampleを抽出 候補数: (2500 + 1500) * 50 = 200,000\n    neg_sample = (\n        pl.read_csv(comp_data_dir / \"nearest_neighbors.csv\")\n        .filter(~pl.col(\"publication_number\").is_in(test_nn_pub))\n        .sample(1500, seed=42)\n        .rename({f'neighbor_{i}':f'target_{i}' for i in range(50)})\n    )\n    test_nn = pl.concat([test_nn, neg_sample]).lazy()\n\n    \n# Filtering only the patent meta-information that appears in the test\nall_pub = test_nn.melt().collect().get_column(\"value\").unique()\nmeta = meta.filter(pl.col(\"publication_number\").is_in(all_pub))\n\nprint('len(meta)', len(meta))\n\nprint('all_pub_len', len(all_pub))\n\n'''\n# Join meta information\npatents = []\nn_unique = meta.select([\"year\", \"month\"]).n_unique()\nfor (year, month), _ in tqdm(meta.group_by([\"year\", \"month\"]), total=n_unique):\n    patent_path = comp_data_dir / f\"patent_data/{year}_{month}.parquet\"\n    # patent = pl.scan_parquet(patent_path).select(pl.exclude([\"description\"]))\n    patent = pl.scan_parquet(patent_path)\n    patents.append(patent)\npatent: pl.LazyFrame = pl.concat(patents)\n\"\"\"\npatent = patent.with_columns(\n    pl.lit(\"\").alias(\"description\"),\n)\n\"\"\"\n\nprint('meta_with_text join')\nmeta_with_text = (\n    meta.lazy().join(patent, on=\"publication_number\", how=\"left\").collect(streaming=True)\n)\n# meta_with_text.write_parquet(\"meta_with_text.parquet\")\n\n# create index\nprint('create index')\ndocuments = meta_with_text.to_dicts()\nPath(\"test_index\").mkdir(parents=True, exist_ok=True)\nwhoosh_utils.create_index(\"test_index\", documents)\n'''\n\n\"\"\"\n# 省メモリ化のために、\nhttps://www.kaggle.com/code/devinanzelmo/uspto-create-validation-index\nに変更\n\"\"\"\n# generate the documents that will go in the index\ndocuments = list()\n# for (year, month), meta_df in tqdm(meta.group_by([\"year\", \"month\"], maintain_order=True)):\nn_unique = meta.select([\"year\", \"month\"]).n_unique()\nfor (year, month), meta_df in tqdm(meta.group_by([\"year\", \"month\"], maintain_order=True), total=n_unique):\n    # for (year, month), meta_df in meta.group_by([\"year\", \"month\"], maintain_order=True):\n    # print((year, month))\n    # meta_df = meta_df.with_columns(pl.col(\"cpc_codes\").list.join(\" \"))\n    meta_df = meta_df.with_columns(pl.col(\"cpc\").list.join(\" \"))\n    \n    patents = pl.read_parquet(comp_data_dir / f\"patent_data/{year}_{month}.parquet\")\n    # patents = pl.scan_parquet(comp_data_dir / f\"patent_data/{year}_{month}.parquet\")\n    \n    patents = patents.filter(pl.col(\"publication_number\").is_in(meta_df[\"publication_number\"]))\n    \n    for i in range(meta_df.shape[0]):\n        \n        d = dict()\n        p = patents.filter(pl.col(\"publication_number\") == meta_df[i, \"publication_number\"])\n        if p.shape[0] > 0:\n            d[\"publication_number\"] = p[0, \"publication_number\"]\n            d[\"title\"] = p[0, \"title\"]\n            d[\"abstract\"] = p[0, \"abstract\"]\n            d[\"claims\"] = p[0, \"claims\"]\n            d[\"description\"] = p[0, \"description\"]\n            # d[\"cpc\"] = meta_df[i, \"cpc_codes\"]\n            d[\"cpc\"] = meta_df[i, \"cpc\"]\n            documents.append(d)\n    \n    del patents\nprint('len(documents)', len(documents))\nPath(\"test_index\").mkdir(parents=True, exist_ok=True)\nwhoosh_utils.create_index(\"test_index\", documents)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}