{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T09:45:29.445039Z","iopub.execute_input":"2024-07-10T09:45:29.445263Z","iopub.status.idle":"2024-07-10T09:45:31.754958Z","shell.execute_reply.started":"2024-07-10T09:45:29.445239Z","shell.execute_reply":"2024-07-10T09:45:31.754015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容\n\n1975年未満も全部使う  \nmax = inf","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh_utils","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_TRAIN = True\nDEBUT_QUERY = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train setup","metadata":{}},{"cell_type":"code","source":"train_idx = whoosh_utils.load_index('/kaggle/input/uspto-explainable-ai/train_index')\nsearcher = whoosh_utils.get_searcher(train_idx)\nqp = whoosh_utils.get_query_parser()\n\nquery = 'ti:balloons OR ti:string'\nwhoosh_utils.execute_query(query, qp, searcher)[:5]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE_FEAT_COLS = ['title']\n# USE_FEAT_COLS = ['abstract']\nUSE_FEAT_COLS = ['title', 'abstract']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search_patent_paths():\n    patent_all_paths = glob.glob('/kaggle/input/uspto-explainable-ai/patent_data/*.parquet')\n\n    patent_paths = []\n    for path in tqdm(patent_all_paths):\n        try:\n            year = int(path.split('/')[-1].split('_')[0])\n        except Exception as e:\n            print(path)\n            continue\n        # if year >= 1975:\n        #     patent_paths.append(path)\n        patent_paths.append(path)\n    \n    print(len(patent_all_paths), len(patent_paths))\n    \n    return patent_paths\n\n\"\"\"\ndef read_train_df(feat_col, path):\n    train_df = pd.read_parquet(path, columns=['publication_number', feat_col])\n    \n    return train_df\n\"\"\"\n\ndef read_train_df(feat_col):\n    patent_all_paths = glob.glob('/kaggle/input/uspto-explainable-ai/patent_data/*.parquet')\n\n    patent_paths = []\n    for path in tqdm(patent_all_paths):\n        try:\n            year = int(path.split('/')[-1].split('_')[0])\n        except Exception as e:\n            print(path)\n            continue\n        # if year >= 1975:\n        #     patent_paths.append(path)\n        patent_paths.append(path)\n    \n    print(len(patent_all_paths), len(patent_paths))\n    \n    dfs = []\n    for path in tqdm(patent_paths):\n        year = int(path.split('/')[-1].split('_')[0])\n        df = pd.read_parquet(path, columns=['publication_number', feat_col])\n        # df['year'] = year\n        dfs.append(df)\n    train_df = pd.concat(dfs).reset_index(drop=True)\n    \n    return train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce memory\ndef document_to_number_seq(train_df, feat_col, word_to_number, prefix_dict):\n    \n    word_set = set()\n    new_documents = []\n    for document in tqdm(train_df[feat_col].values):\n        # for word in document.split():\n        document = [token.text for token in custom_analyzer(document)]\n        document = [f'{prefix_dict[feat_col]}:{word}' for word in document]\n        \n        new_documents.append(document)\n        for word in document:\n            word_set.add(word)\n    \n    word_set = sorted(list(word_set))\n    print('word_set len:', len(word_set))\n    # word_to_number = dict()\n    start_idx = len(word_to_number)\n    for i, word in enumerate(word_set):\n        # word_to_number[word] = str(start_idx + i)\n        word_to_number[word] = start_idx + i\n    number_seq_list = []\n    for document in tqdm(new_documents):\n        number_seq = []\n        for word in document:\n            number_seq.append(word_to_number[word])\n    \n        # new_document = ' '.join(new_document)\n    \n        number_seq_list.append(number_seq)\n    \n    return number_seq_list, word_to_number","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# main","metadata":{}},{"cell_type":"code","source":"patent_metadata_df = pd.read_parquet('/kaggle/input/uspto-explainable-ai/patent_metadata.parquet', columns=['publication_number', 'publication_date', 'cpc_codes'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nprint(len(patent_metadata_df))\n# use_idxs = patent_metadata_df['year'] >= 1975\nuse_idxs = patent_metadata_df['publication_date'].dt.year >= 1975\npatent_metadata_df = patent_metadata_df[use_idxs].reset_index(drop=True)\nprint(len(patent_metadata_df))\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh, re\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\n\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t\n\nBRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n        'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\n\ncustom_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def change_cpc_codes(patent_metadata_df):\n    prefix_dict = {\n        'title': 'ti',\n        'abstract': 'ab',\n        'claims': 'clm',\n        'description': 'detd'\n    }\n    \n    word_set = set()\n    for cpc_codes in patent_metadata_df['cpc_codes'].values:\n        for c in cpc_codes:\n            word_set.add('cpc:' + c)\n\n    word_set = sorted(list(word_set))\n    print('word_set len:', len(word_set))\n    \n    word_to_number = dict()\n    for i, word in enumerate(word_set):\n        # word_to_number[word] = str(i)\n        word_to_number[word] = i\n        \n    new_cpc_codes = [[word_to_number['cpc:' + c] for c in cpc_codes] for cpc_codes in patent_metadata_df['cpc_codes'].values]\n    \n    for feat_col in USE_FEAT_COLS:\n        train_df = read_train_df(feat_col)\n        \n        number_seq, word_to_number = document_to_number_seq(train_df, feat_col, word_to_number, prefix_dict)\n        # train_df[feat_col] = number_seq\n\n        pub_to_feat = dict(zip(train_df['publication_number'], number_seq))\n\n        for i, pub in enumerate(tqdm(patent_metadata_df['publication_number'].values, total=len(patent_metadata_df))):\n            if pub not in pub_to_feat:\n                continue\n\n            feats = pub_to_feat[pub]\n            if len(feats) == 0:\n                continue\n                \n            # tokenizerあり 小文字化とかもあるはずなので、入れるべき? 正確に比較できていないけど、大きく変わらなそうなので入れない\n            # feats = [token.text for token in custom_analyzer(feats)]\n            # feats = sorted(list(set(feats)))\n            \n            # tokenizerなし\n            # feats = list(set(feats.split()))\n\n            # for feat in feats.split():\n            for feat in feats:\n                # if 'id:' in feat: # number化したなら外す\n                #     continue\n                \n                # new_cpc_codes[i].append(f'{prefix_dict[feat_col]}:{feat}')\n                new_cpc_codes[i].append(feat)\n    \n    debug_idx = 0\n    print(new_cpc_codes[debug_idx], set(new_cpc_codes[debug_idx]))\n    \n    patent_metadata_df['cpc_codes'] = new_cpc_codes\n    \n    return patent_metadata_df, word_to_number","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patent_metadata_df, word_to_number = change_cpc_codes(patent_metadata_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(word_to_number)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_to_word = dict()\nfor word, number in word_to_number.items():\n    number_to_word[number] = word","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_cpc_to_pub_set(patent_metadata_df):\n    cpc_to_pub_set = defaultdict(set)\n    cpc_to_pubcount = defaultdict(int)\n    \n    for pub, cpc_codes in tqdm(patent_metadata_df[['publication_number', 'cpc_codes']].values):\n        for cpc in cpc_codes:\n            cpc_to_pub_set[cpc].add(pub)\n    \n    cpc_to_pub_set_new = defaultdict(set)\n    for cpc, pub_set in cpc_to_pub_set.items():\n        # if len(pub_set) <= 1000:\n        # if len(pub_set) <= 100000:\n        cpc_to_pub_set_new[cpc] = pub_set\n    cpc_to_pub_set = cpc_to_pub_set_new\n    \n    counts = []\n    for cpc, pub_set in cpc_to_pub_set.items():\n        cpc_to_pubcount[cpc] = len(pub_set)\n        counts.append(len(pub_set))\n    \n    print(pd.DataFrame(counts).describe())\n    \n    publication_to_cpc = defaultdict(list)\n    for cpc, pub_set in cpc_to_pub_set.items():\n        for pub in pub_set:\n            publication_to_cpc[pub].append(cpc)\n    \n    return cpc_to_pub_set, cpc_to_pubcount, publication_to_cpc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpc_to_pub_set, cpc_to_pubcount, publication_to_cpc = make_cpc_to_pub_set(patent_metadata_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del patent_metadata_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pkl化するもの\n\nword_to_number, number_to_word, cpc_to_pub_set, cpc_to_pubcount, publication_to_cpc","metadata":{}},{"cell_type":"code","source":"import pickle","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_number, open(f'word_to_number.pkl', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(number_to_word, open(f'number_to_word.pkl', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(cpc_to_pub_set, open(f'cpc_to_pub_set.pkl', 'wb'))\npickle.dump(cpc_to_pubcount, open(f'cpc_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_cpc, open(f'publication_to_cpc.pkl', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}