{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8914164,"sourceType":"datasetVersion","datasetId":5360501},{"sourceId":9008587,"sourceType":"datasetVersion","datasetId":5427363},{"sourceId":9009428,"sourceType":"datasetVersion","datasetId":5428005},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":187159588,"sourceType":"kernelVersion"},{"sourceId":188357773,"sourceType":"kernelVersion"},{"sourceId":189039731,"sourceType":"kernelVersion"},{"sourceId":189045602,"sourceType":"kernelVersion"},{"sourceId":189061218,"sourceType":"kernelVersion"},{"sourceId":189232069,"sourceType":"kernelVersion"},{"sourceId":189294219,"sourceType":"kernelVersion"},{"sourceId":189579978,"sourceType":"kernelVersion"}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os\nimport shutil\n\nimport whoosh_utils","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:19.852532Z","iopub.execute_input":"2024-07-24T12:33:19.85285Z","iopub.status.idle":"2024-07-24T12:33:53.817615Z","shell.execute_reply.started":"2024-07-24T12:33:19.852825Z","shell.execute_reply":"2024-07-24T12:33:53.816635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy as cp","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:53.819571Z","iopub.execute_input":"2024-07-24T12:33:53.820396Z","iopub.status.idle":"2024-07-24T12:33:55.246857Z","shell.execute_reply.started":"2024-07-24T12:33:53.820363Z","shell.execute_reply":"2024-07-24T12:33:55.246115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\na1 = np.arange(100000)\na2 = np.arange(100000)\nfor _ in tqdm(range(100000)):\n    d = np.intersect1d(a1, a2)\n    \na1 = cp.arange(100000)\na2 = cp.arange(100000)\nfor _ in tqdm(range(100000)):\n    d = cp.intersect1d(a1, a2)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.247929Z","iopub.execute_input":"2024-07-24T12:33:55.248212Z","iopub.status.idle":"2024-07-24T12:33:55.256502Z","shell.execute_reply.started":"2024-07-24T12:33:55.248188Z","shell.execute_reply":"2024-07-24T12:33:55.255523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容  \n\nnegpick == 0 のときだけ候補にする  \ncpcを一番最後にして、ワイルドカードを使うのをできるだけ減らす  \ncpc、使うのを1つまでにする  \n近傍の近傍とか、不要なものを消す\n\n\n\n\n","metadata":{}},{"cell_type":"code","source":"\n# TRAIN_PKL_PATH = '/kaggle/input/sa-cpc-title-abst-clm-desc-10-5-use-allpub-svpkl/'\n# TRAIN_PKL_PATH = ''\n\n# TRAIN_PKL_PATH = '/kaggle/input/sa-cpc-title-abst-clm-inf-desc-10-5-save-pkl/'\n\nTRAIN_PKL_PATH = '/kaggle/input/sa-cpc-title-abst-clm-inf-cupy-save/'","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.257545Z","iopub.execute_input":"2024-07-24T12:33:55.257812Z","iopub.status.idle":"2024-07-24T12:33:55.265244Z","shell.execute_reply.started":"2024-07-24T12:33:55.257791Z","shell.execute_reply":"2024-07-24T12:33:55.264443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_TRAIN = False\nIS_REDUCE_TRAIN = False\n\nIS_ADD_NEG_0_PATTERN = True","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.26737Z","iopub.execute_input":"2024-07-24T12:33:55.267651Z","iopub.status.idle":"2024-07-24T12:33:55.274718Z","shell.execute_reply.started":"2024-07-24T12:33:55.267629Z","shell.execute_reply":"2024-07-24T12:33:55.273849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEG_MAX_COUNT = 1000\nPUB_MAX_COUNT = 50 + NEG_MAX_COUNT # 10\nSUB_MAX_COUNT = 50 + NEG_MAX_COUNT # 100\n\nPATTERN_NUM_MAX = 10000\n\nCHAR_LIMIT = 9000","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.275781Z","iopub.execute_input":"2024-07-24T12:33:55.276082Z","iopub.status.idle":"2024-07-24T12:33:55.284056Z","shell.execute_reply.started":"2024-07-24T12:33:55.276055Z","shell.execute_reply":"2024-07-24T12:33:55.283307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEG_WEIGHT = 0.01 # 0.1","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.28498Z","iopub.execute_input":"2024-07-24T12:33:55.285238Z","iopub.status.idle":"2024-07-24T12:33:55.293329Z","shell.execute_reply.started":"2024-07-24T12:33:55.285217Z","shell.execute_reply":"2024-07-24T12:33:55.292533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"T0 = 1\nT1 = 0.1\n\nmax_time = 1 # 5","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.294267Z","iopub.execute_input":"2024-07-24T12:33:55.294554Z","iopub.status.idle":"2024-07-24T12:33:55.302535Z","shell.execute_reply.started":"2024-07-24T12:33:55.294531Z","shell.execute_reply":"2024-07-24T12:33:55.301779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN:\n    nn_path = '/kaggle/input/create-valid-index-add-claim/nn_df_for_index.csv'\n    nn_df = pd.read_csv(nn_path)\n    print('len(nn_df)', len(nn_df))\n    \n    nn_df.to_csv('nn_df_for_index.csv', index=False)\nelse:\n    nn_df = pd.read_csv('/kaggle/input/uspto-explainable-ai/test.csv')\n    nn_df.to_csv('nn_df_for_index.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.303367Z","iopub.execute_input":"2024-07-24T12:33:55.303655Z","iopub.status.idle":"2024-07-24T12:33:55.526439Z","shell.execute_reply.started":"2024-07-24T12:33:55.303633Z","shell.execute_reply":"2024-07-24T12:33:55.525534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.makedirs('nn_nn', exist_ok=True)\nos.makedirs('reduce', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.527641Z","iopub.execute_input":"2024-07-24T12:33:55.527926Z","iopub.status.idle":"2024-07-24T12:33:55.532834Z","shell.execute_reply.started":"2024-07-24T12:33:55.527902Z","shell.execute_reply":"2024-07-24T12:33:55.531876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# publication to word、nn_df以外のpubは消す","metadata":{}},{"cell_type":"code","source":"%%writefile nn_nn/publication_to_word.py\n\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os\n\nnn_df = pd.read_csv('nn_df_for_index.csv')\n\nuse_pub = set()\nfor nn in nn_df.values:\n    for n in nn:\n        use_pub.add(n)\n\nprint(len(use_pub))\n\n# publication_to_word = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n\npublication_to_word = {}\nfor i in tqdm(range(21)):\n    _publication_to_word = pickle.load(open(f'/kaggle/input/cpc-title-abst-claim-inf-chunk/publication_to_word_{i}.pkl', 'rb'))\n    for pub, word in _publication_to_word.items():\n        if pub in use_pub:\n            publication_to_word[pub] = word\n    \nprint('len(publication_to_word)', len(publication_to_word))\n\npickle.dump(publication_to_word, open(f'reduce/publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.533946Z","iopub.execute_input":"2024-07-24T12:33:55.534199Z","iopub.status.idle":"2024-07-24T12:33:55.544086Z","shell.execute_reply.started":"2024-07-24T12:33:55.534177Z","shell.execute_reply":"2024-07-24T12:33:55.543191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN and TRAIN_PKL_PATH != '':\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/publication_to_word.pkl', 'reduce/publication_to_word.pkl')\nelse:\n    !python nn_nn/publication_to_word.py","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:55.545281Z","iopub.execute_input":"2024-07-24T12:33:55.545548Z","iopub.status.idle":"2024-07-24T12:33:57.747879Z","shell.execute_reply.started":"2024-07-24T12:33:55.545516Z","shell.execute_reply":"2024-07-24T12:33:57.746918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# publication to int  \n","metadata":{}},{"cell_type":"code","source":"%%writefile nn_nn/pub_to_num.py\n\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os\n\npatent_metadata_df = pd.read_parquet('/kaggle/input/uspto-explainable-ai/patent_metadata.parquet',\n                                     columns=['publication_number'])\n\npub_to_num = dict()\nnum_to_pub = dict()\nfor num, pub in enumerate(patent_metadata_df['publication_number'].values):\n    pub_to_num[pub] = num\n    num_to_pub[num] = pub\n\npickle.dump(pub_to_num, open(f'reduce/pub_to_num.pkl', 'wb'))\npickle.dump(num_to_pub, open(f'reduce/num_to_pub.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:57.749123Z","iopub.execute_input":"2024-07-24T12:33:57.749479Z","iopub.status.idle":"2024-07-24T12:33:57.756166Z","shell.execute_reply.started":"2024-07-24T12:33:57.749443Z","shell.execute_reply":"2024-07-24T12:33:57.755315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN and TRAIN_PKL_PATH != '':\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/pub_to_num.pkl', 'reduce/pub_to_num.pkl')\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/num_to_pub.pkl', 'reduce/num_to_pub.pkl')\nelse:\n    !python nn_nn/pub_to_num.py","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:33:57.760651Z","iopub.execute_input":"2024-07-24T12:33:57.76094Z","iopub.status.idle":"2024-07-24T12:34:04.720311Z","shell.execute_reply.started":"2024-07-24T12:33:57.760906Z","shell.execute_reply":"2024-07-24T12:34:04.719196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# word to publication、nn_df以外のwordは消す  \n","metadata":{}},{"cell_type":"code","source":"os.makedirs('word_to_pub_set_chunk', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:34:04.721888Z","iopub.execute_input":"2024-07-24T12:34:04.722241Z","iopub.status.idle":"2024-07-24T12:34:04.729738Z","shell.execute_reply.started":"2024-07-24T12:34:04.722209Z","shell.execute_reply":"2024-07-24T12:34:04.728875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile nn_nn/reduce_pkl.py\n\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os\n\nbase_path = 'reduce/'\npub_to_num = pickle.load(open(base_path + 'pub_to_num.pkl', 'rb'))\nnum_to_pub = pickle.load(open(base_path + 'num_to_pub.pkl', 'rb'))\n\nnn_df = pd.read_csv('nn_df_for_index.csv')\n\nuse_pub = set()\nfor nn in nn_df.values:\n    for n in nn:\n        use_pub.add(pub_to_num[n])\n\nprint(len(use_pub))\n\npublication_to_word = pickle.load(open('reduce/' + 'publication_to_word.pkl', 'rb'))\npublication_to_word_new = defaultdict(list)\nfor pub, word in tqdm(publication_to_word.items()):\n    publication_to_word_new[pub_to_num[pub]] = word\npublication_to_word = publication_to_word_new\n\n\nbase_path = '/kaggle/input/cpc-title-abst-claim-inf/'\nword_to_pubcount = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\nuse_words = set()\nfor pub in tqdm(use_pub):\n    words = publication_to_word[pub]\n    use_words.update(words)\nprint('len(use_words)', len(use_words))\n\n\nfor i in tqdm(range(21)):\n    print(f'chunk_{i}')\n    word_to_pub_set = {}\n    _word_to_pub_set = pickle.load(open(f'/kaggle/input/cpc-title-abst-claim-inf-chunk/word_to_pub_set_{i}.pkl', 'rb'))\n    for word, pub_set in _word_to_pub_set.items():\n        if word in use_words:            \n            pub_set = np.array([pub_to_num[pub] for pub in pub_set], dtype='int32')\n            word_to_pub_set[word] = pub_set\n\n    pickle.dump(word_to_pub_set, open(f'reduce/word_to_pub_set_chunk_{i}.pkl', 'wb'))\n    \n    del word_to_pub_set, _word_to_pub_set\n    gc.collect()\n\ngc.collect()\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\nremove_words = [w for w in number_to_word.keys() if w not in use_words]\nprint('len(remove_words)', len(remove_words))\nfor number in remove_words:\n    word = number_to_word[number]\n    word_to_number.pop(word)\n    number_to_word.pop(number)\n\nprint('len(word_to_number)', len(word_to_number))\nprint('len(number_to_word)', len(number_to_word))\n\n\nremove_words = [w for w in word_to_pubcount.keys() if w not in use_words]\nprint('len(remove_words)', len(remove_words))\nfor word in remove_words:\n    word_to_pubcount.pop(word)\nprint('len(word_to_pubcount)', len(word_to_pubcount))\n\npickle.dump(word_to_number, open(f'reduce/word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(f'reduce/number_to_word.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(f'reduce/word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(f'reduce/publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:34:04.732365Z","iopub.execute_input":"2024-07-24T12:34:04.732691Z","iopub.status.idle":"2024-07-24T12:34:04.746317Z","shell.execute_reply.started":"2024-07-24T12:34:04.732668Z","shell.execute_reply":"2024-07-24T12:34:04.744596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN and TRAIN_PKL_PATH != '':\n    # shutil.copy(f'{TRAIN_PKL_PATH}reduce/word_to_pub_set.pkl', 'reduce/word_to_pub_set.pkl')\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/word_to_number.pkl', 'reduce/word_to_number.pkl')\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/number_to_word.pkl', 'reduce/number_to_word.pkl')\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/word_to_pubcount.pkl', 'reduce/word_to_pubcount.pkl')\n    shutil.copy(f'{TRAIN_PKL_PATH}reduce/publication_to_word.pkl', 'reduce/publication_to_word.pkl')\nelse:\n    !python nn_nn/reduce_pkl.py","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:34:04.748351Z","iopub.execute_input":"2024-07-24T12:34:04.748714Z","iopub.status.idle":"2024-07-24T12:34:05.825366Z","shell.execute_reply.started":"2024-07-24T12:34:04.748679Z","shell.execute_reply":"2024-07-24T12:34:05.824295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load index ","metadata":{}},{"cell_type":"code","source":"if IS_TRAIN:\n    # train_idx = whoosh_utils.load_index('/kaggle/input/create-valid-index-add-claim/test_index')\n    train_idx = whoosh_utils.load_index('/kaggle/input/uspto-test-index/test_index')\n    searcher = whoosh_utils.get_searcher(train_idx)\n    qp = whoosh_utils.get_query_parser()\n\n    # query = 'ti:balloons OR ti:string'\n    query = '(cpc:A01H6/42 ab:cuttings) OR (ab:regal-ab:rooting) OR (ab:magenta-ab:umbels) OR (ti:geranium-ti:variety-ab:cherry-ab:foliage-ab:pink-clm:geranium) OR (ti:bravo-ab:bicolored) OR (ab:foliage-ab:geranium-ab:regal-clm:geranium-detd:duchess) OR (ti:debutante-ti:pelargonium) OR (ti:geranium-ab:bright-ab:garden-ab:geranium-ab:mounded-ab:zonal-clm:geranium-clm:oglger14007-clm:zonal-detd:oglger14007) OR (ti:geranium-ti:variety-ab:blossoms-ab:foliage-ab:orchid-clm:geranium) OR (ti:louise-ab:ivy) OR (ti:hots) OR (ti:geranium-ti:variety-ab:orange-ab:rain-clm:geranium)'\n    print(whoosh_utils.execute_query(query, qp, searcher)[:5])","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:34:05.826442Z","iopub.execute_input":"2024-07-24T12:34:05.826745Z","iopub.status.idle":"2024-07-24T12:35:06.064913Z","shell.execute_reply.started":"2024-07-24T12:34:05.826721Z","shell.execute_reply":"2024-07-24T12:35:06.064017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 事前に前処理したものを読み込む  \n","metadata":{}},{"cell_type":"code","source":"if IS_TRAIN and TRAIN_PKL_PATH != '':\n    chunk_path = TRAIN_PKL_PATH + 'reduce/'\nelse:\n    chunk_path = 'reduce/'\nword_to_pub_set = {}\nfor i in tqdm(range(21)):\n    _word_to_pub_set = pickle.load(open(chunk_path + f'word_to_pub_set_chunk_{i}.pkl', 'rb'))\n    for k, v in _word_to_pub_set.items():\n        # word_to_pub_set[k] = v\n        word_to_pub_set[k] = cp.array(np.sort(v), dtype=cp.int32)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:35:06.066175Z","iopub.execute_input":"2024-07-24T12:35:06.06646Z","iopub.status.idle":"2024-07-24T12:37:42.138019Z","shell.execute_reply.started":"2024-07-24T12:35:06.066435Z","shell.execute_reply":"2024-07-24T12:37:42.137127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport pickle\n\n\nbase_path = 'reduce/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:42.139864Z","iopub.execute_input":"2024-07-24T12:37:42.14015Z","iopub.status.idle":"2024-07-24T12:37:43.084434Z","shell.execute_reply.started":"2024-07-24T12:37:42.140126Z","shell.execute_reply":"2024-07-24T12:37:43.083545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pubcount = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:43.087585Z","iopub.execute_input":"2024-07-24T12:37:43.087941Z","iopub.status.idle":"2024-07-24T12:37:43.423578Z","shell.execute_reply.started":"2024-07-24T12:37:43.087917Z","shell.execute_reply":"2024-07-24T12:37:43.422732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\n\n# word_to_pub_set = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:43.424681Z","iopub.execute_input":"2024-07-24T12:37:43.424955Z","iopub.status.idle":"2024-07-24T12:37:43.430904Z","shell.execute_reply.started":"2024-07-24T12:37:43.424932Z","shell.execute_reply":"2024-07-24T12:37:43.429929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"publication_to_word = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:43.43207Z","iopub.execute_input":"2024-07-24T12:37:43.432395Z","iopub.status.idle":"2024-07-24T12:37:45.136601Z","shell.execute_reply.started":"2024-07-24T12:37:43.432349Z","shell.execute_reply":"2024-07-24T12:37:45.135746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub_to_num = pickle.load(open(base_path + 'pub_to_num.pkl', 'rb'))\nnum_to_pub = pickle.load(open(base_path + 'num_to_pub.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:45.137957Z","iopub.execute_input":"2024-07-24T12:37:45.138248Z","iopub.status.idle":"2024-07-24T12:37:56.841227Z","shell.execute_reply.started":"2024-07-24T12:37:45.138224Z","shell.execute_reply":"2024-07-24T12:37:56.840435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = []\nfor _, cnt in word_to_pubcount.items():\n    counts.append(cnt)\nplt.hist(counts)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:37:56.842351Z","iopub.execute_input":"2024-07-24T12:37:56.842703Z","iopub.status.idle":"2024-07-24T12:38:02.117071Z","shell.execute_reply.started":"2024-07-24T12:37:56.842671Z","shell.execute_reply":"2024-07-24T12:38:02.116175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_df = pd.read_csv('nn_df_for_index.csv')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:38:02.118507Z","iopub.execute_input":"2024-07-24T12:38:02.118813Z","iopub.status.idle":"2024-07-24T12:38:02.198326Z","shell.execute_reply.started":"2024-07-24T12:38:02.118789Z","shell.execute_reply":"2024-07-24T12:38:02.197596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(nn_df.values.shape[0]):\n    for j in range(nn_df.values.shape[1]):\n        nn_df.values[i, j] = pub_to_num[nn_df.values[i, j]]","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:38:02.199627Z","iopub.execute_input":"2024-07-24T12:38:02.199912Z","iopub.status.idle":"2024-07-24T12:38:03.085316Z","shell.execute_reply.started":"2024-07-24T12:38:02.199889Z","shell.execute_reply":"2024-07-24T12:38:03.084319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neighbors = nn_df.values[:, 1:]","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:38:03.086523Z","iopub.execute_input":"2024-07-24T12:38:03.086814Z","iopub.status.idle":"2024-07-24T12:38:03.091445Z","shell.execute_reply.started":"2024-07-24T12:38:03.08679Z","shell.execute_reply":"2024-07-24T12:38:03.09046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nword_pattern_to_pub_set_list = [defaultdict(set) for _ in range(len(neighbors))]\nword_pattern_to_pub_set_all = dict()\nadd_count_list = []\n\n\nfor i in tqdm(range(len(neighbors))):\n\n    word_pattern_to_pub_set = word_pattern_to_pub_set_list[i]\n    add_list = []\n    \n\n    nn_list = neighbors[i]\n    # nn_array = np.array(neighbors[i], dtype=np.int32)\n    \n    nn_array = neighbors[i]\n    nn_array = np.sort(nn_array)\n    nn_array = cp.array(nn_array, dtype=np.int32)\n    nn_set = set(nn_list)\n\n    # use_words_seen = set()\n    use_words_len_max = -1\n\n    # 行ごとにcacheをリセットして、メモリが増えすぎないようにする\n    pub_set_cache = {}\n    use_words_cache = {}\n    \n    t_sum = 0\n    for r in range(1, 3):\n        debug_count=0\n        for nn_comb in itertools.combinations(nn_list, r):\n            counts = []\n            if nn_comb[0] not in use_words_cache:\n                use_words_cache[nn_comb[0]] = set([w for w in publication_to_word[nn_comb[0]]])\n            use_words = use_words_cache[nn_comb[0]].copy()\n            for nn in nn_comb[1:]:\n                if not nn in use_words_cache:\n                    use_words_cache[nn] = set([w for w in publication_to_word[nn]])\n                use_words &= use_words_cache[nn]\n                \n            use_words = tuple(sorted(list(use_words)))\n\n            if len(use_words) == 0: #  or use_words in use_words_seen\n                continue\n\n            use_words_sorted = sorted(use_words, key=lambda x: word_to_pubcount[x])\n            \n            use_words = tuple([w for w in use_words if w in set(use_words_sorted)])\n            # use_words_len_max = max(len(use_words), use_words_len_max)\n            \n            use_words_final = []\n            \"\"\"\n            # 要素数1万以上の集合は、setを作る時間の短縮のために事前にsetにしていた。今は不要\n            if isinstance(word_to_pub_set[use_words_sorted[0]], set):\n                all_pub_set = word_to_pub_set[use_words_sorted[0]].copy()\n            else:\n                if not use_words_sorted[0] in pub_set_cache:\n                    pub_set_cache[use_words_sorted[0]] = set(word_to_pub_set[use_words_sorted[0]])\n                all_pub_set = pub_set_cache[use_words_sorted[0]].copy()\n            \"\"\"\n            all_pub_set = word_to_pub_set[use_words_sorted[0]]\n            \n            use_words_final.append(use_words_sorted[0])\n            \n            cpc_count = 1 if number_to_word[use_words_sorted[0]][:3] == 'cpc' else 0\n            \n            \n            if not len(all_pub_set) <= r:\n                for word in use_words_sorted[1:]:\n                    \n                    if number_to_word[word][:3] == 'cpc' and cpc_count == 1:\n                        continue\n                    \n                    before_len = len(all_pub_set)\n                    \"\"\"\n                    if isinstance(word_to_pub_set[word], set):\n                        all_pub_set &= word_to_pub_set[word]\n                    else:\n                        if not word in pub_set_cache:\n                            pub_set_cache[word] = set(word_to_pub_set[word])\n                        all_pub_set &= pub_set_cache[word]\n                    \"\"\"\n                    \n\n                    # all_pub_set = np.intersect1d(all_pub_set, word_to_pub_set[word], assume_unique = True)\n                    all_pub_set = cp.intersect1d(all_pub_set, word_to_pub_set[word], assume_unique = True)\n                    \n                    \n                    after_len = len(all_pub_set)\n                    \n                    # 集合の要素数が減ったときだけ追加\n                    if after_len < before_len:\n                        use_words_final.append(word)\n                        \n                        if number_to_word[word][:3] == 'cpc':\n                            cpc_count += 1\n\n                    # cpcのワイルドカードを使わないようにしたので、↓はいらない\n                    \"\"\"\n                    else:\n                        # cpcでなければ追加\n                        if number_to_word[word][:3] != 'cpc':\n                            use_words_final.append(word)\n                    \"\"\"\n                    # use_words_final.append(word)\n                    \n                    counts.append(len(all_pub_set))\n\n                    # nn_combだけになったら終了\n                    if len(all_pub_set) <= r: \n                        break\n                    \n                    # 近傍だけになったら終了\n                    cnt1 = len(all_pub_set)\n                    \n                    # cnt2 = len(all_pub_set & nn_set)\n                    # cnt2 = len(np.intersect1d(all_pub_set, nn_array, assume_unique = True))\n                    \n                    if cnt1 <= 50:                 \n                        t1 = time.time()\n                        cnt2 = len(cp.intersect1d(all_pub_set, nn_array, assume_unique = True))\n                        # cnt2 = cp.in1d(all_pub_set ,nn_array, assume_unique = True)\n                        # cnt2 = cnt2.sum()\n\n                        t_sum += time.time() - t1\n\n                        if cnt1 == cnt2:\n                            break\n                    \n           \n            \n            \n            use_words = tuple(use_words_final)\n            use_words_len_max = max(len(use_words), use_words_len_max)\n\n            if debug_count < -1:\n                print(counts)\n\n            debug_count += 1\n            if len(all_pub_set) <= 50: # len(all_pub_set) <= 50なしだと、多分OOM + 間に合わない\n                \n            \n                all_pub_set = cp.asnumpy(all_pub_set)\n\n                \n                nn_comb_set = set(nn_comb)\n                all_pub_set = set(all_pub_set)\n                # nn_setで、追加できるなら追加\n                add_set = all_pub_set & nn_set\n                nn_comb_set = nn_comb_set | add_set\n                \n                # neg_pub_set = all_pub_set - nn_comb_set\n\n                # if len(neg_pub_set) == 0:\n                if len(nn_comb_set) == len(all_pub_set): # negative=0なら追加\n                    add_list.append((use_words, nn_comb_set, all_pub_set))\n\n        if i <= 50:\n            print(t_sum)\n            print(f'r >= {r} len(add_list): {len(add_list)}')        \n            print('use_words_len_max ', use_words_len_max )\n\n    add_count = 0\n    for word_pattern, pub_set, all_pub_set in add_list:\n        if word_pattern in word_pattern_to_pub_set:\n            continue\n\n        word_pattern_to_pub_set[word_pattern] = pub_set\n        word_pattern_to_pub_set_all[word_pattern] = all_pub_set\n\n        add_count += 1\n\n    if i <= 50:\n        print(f'len(add_list): {len(add_list)}, add_count: {add_count}')\n    add_count_list.append(add_count)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:30.353849Z","iopub.execute_input":"2024-07-24T13:00:30.354584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pd.DataFrame(add_count_list).describe())","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:27.685646Z","iopub.execute_input":"2024-07-24T12:42:27.686018Z","iopub.status.idle":"2024-07-24T12:42:27.706568Z","shell.execute_reply.started":"2024-07-24T12:42:27.68599Z","shell.execute_reply":"2024-07-24T12:42:27.705632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef count_query_len(word_pattern):\n    is_replace_AND = False\n    prefix_set = set([number_to_word[word].split(':')[0] for word in word_pattern])\n    if 'cpc' not in prefix_set:\n        is_replace_AND = True\n    \n    if len(word_pattern) == 1 or is_replace_AND:\n        token_len = 2\n    else:\n        token_len = 3\n    return token_len\n\"\"\"\n# cpcもtoken_len=2でできるようになったので、変更\ndef count_query_len(word_pattern):\n    token_len = 2 \n    return token_len","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:27.914963Z","iopub.execute_input":"2024-07-24T12:42:27.915341Z","iopub.status.idle":"2024-07-24T12:42:27.92041Z","shell.execute_reply.started":"2024-07-24T12:42:27.91531Z","shell.execute_reply":"2024-07-24T12:42:27.919543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filtering_word_pattern_to_pub_set_list(word_pattern_to_pub_set_list):\n    \n    word_pattern_to_pub_set_list_new = []\n    # for word_pattern_to_pub_set in word_pattern_to_pub_set_list:\n    for sample_idx, word_pattern_to_pub_set in enumerate(word_pattern_to_pub_set_list):\n\n        word_pattern_to_pub_set_new = defaultdict(set)\n        sort_keys = []\n\n        # データの処理とソートキーの生成\n        for word_pattern, pub_set in word_pattern_to_pub_set.items():\n            max_count = len(word_pattern_to_pub_set_all[word_pattern])\n            if max_count <= PUB_MAX_COUNT:\n                # word_pattern_to_pub_set_new[word_pattern] = pub_set\n\n                neg_pub_set = word_pattern_to_pub_set_all[word_pattern] - pub_set                \n                neg_weight = len(neg_pub_set)\n\n                sort_key = (len(pub_set), -count_query_len(word_pattern), -neg_weight)\n                sort_keys.append((sort_key, word_pattern))\n                word_pattern_to_pub_set_new[word_pattern] = pub_set\n\n        # ソートキーでソート\n        sorted_word_patterns = sorted(sort_keys, key=lambda x: x[0], reverse=True)\n\n        # ソート結果に基づいて新しい辞書を作成\n        word_pattern_to_pub_set_new = [(word_pattern, word_pattern_to_pub_set_new[word_pattern]) for _, word_pattern in sorted_word_patterns]\n        \n        # 重複処理 \n        seen = set()\n        new_state = []\n        for word_pattern, pub_set in word_pattern_to_pub_set_new:\n            # pattern = tuple(sorted(list(pub_set)))\n            pattern = (tuple(sorted(list(pub_set))), count_query_len(word_pattern))\n            if pattern in seen:\n                continue\n            seen.add(pattern)\n            new_state.append((word_pattern, pub_set))\n        word_pattern_to_pub_set_new = new_state\n        \n        \n        # (検索結果内のtarget数、検索結果内の全てのpublication_number数)でソートした上位 PATTERN_NUM_MAX 件\n        word_pattern_to_pub_set_new = word_pattern_to_pub_set_new[:PATTERN_NUM_MAX]\n            \n        # この処理いらない。new_stateと同じデータ構造なので。\n        word_pattern_to_pub_set = []\n        for word_pattern, pub_set in word_pattern_to_pub_set_new:\n            word_pattern_to_pub_set.append((word_pattern, pub_set))\n        \n        word_pattern_to_pub_set_list_new.append(word_pattern_to_pub_set)\n\n    word_pattern_to_pub_set_list = word_pattern_to_pub_set_list_new\n    \n    return word_pattern_to_pub_set_list","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:28.162809Z","iopub.execute_input":"2024-07-24T12:42:28.163754Z","iopub.status.idle":"2024-07-24T12:42:28.17427Z","shell.execute_reply.started":"2024-07-24T12:42:28.163708Z","shell.execute_reply":"2024-07-24T12:42:28.173375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_pattern_to_pub_set_list = filtering_word_pattern_to_pub_set_list(word_pattern_to_pub_set_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:28.857237Z","iopub.execute_input":"2024-07-24T12:42:28.857643Z","iopub.status.idle":"2024-07-24T12:42:29.674722Z","shell.execute_reply.started":"2024-07-24T12:42:28.857612Z","shell.execute_reply":"2024-07-24T12:42:29.673921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recall_list = []\nfor word_pattern_to_pub_set in word_pattern_to_pub_set_list:\n    true_set = set()\n    # for word_pattern, true_pub_set in word_pattern_to_pub_set.items():\n    for word_pattern, true_pub_set in word_pattern_to_pub_set:\n        true_set |= true_pub_set\n    recall_list.append(len(true_set))\n\nprint(pd.DataFrame(recall_list).describe())\nplt.hist(recall_list)\nplt.title('recall distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:29.676196Z","iopub.execute_input":"2024-07-24T12:42:29.676479Z","iopub.status.idle":"2024-07-24T12:42:30.031216Z","shell.execute_reply.started":"2024-07-24T12:42:29.676455Z","shell.execute_reply":"2024-07-24T12:42:30.030263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cpcの組み合わせ数\nlis = []\nfor word_pattern_to_pub_set in word_pattern_to_pub_set_list:\n    lis.append(len(word_pattern_to_pub_set))\nprint(pd.DataFrame(lis).describe())\nplt.hist(lis)\nplt.title('combincation count')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:30.032675Z","iopub.execute_input":"2024-07-24T12:42:30.032946Z","iopub.status.idle":"2024-07-24T12:42:30.337258Z","shell.execute_reply.started":"2024-07-24T12:42:30.032922Z","shell.execute_reply":"2024-07-24T12:42:30.336282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import total_ordering\n\n@total_ordering\nclass State:\n    def __init__(self, use_word_pattern_list, not_use_word_pattern_list, use_true_pub_set, neg_count, score=None,\n                 is_hard_penalty=False):\n        self.use_word_pattern_list = use_word_pattern_list\n        self.not_use_word_pattern_list = not_use_word_pattern_list\n        # self.use_pub_set = use_pub_set\n        self.use_true_pub_set = use_true_pub_set\n        self.neg_count = neg_count\n        \n        # 簡単にrecall上げられるサンプルなら、negativeの重みを強める\n        self.is_hard_penalty = is_hard_penalty\n    \n    def __lt__(self, other):\n        return self.score < other.score\n\n    def __eq__(self, other):\n        return self.score == other.score\n    \n    def calc_score(self):\n        \n        # self.score = SUB_MAX_COUNT * (len(self.use_true_pub_set) - 0.25 * self.neg_count)\n        \n        if self.is_hard_penalty:\n            # self.score = len(self.use_true_pub_set) - 0.25 * self.neg_count\n            self.score = len(self.use_true_pub_set) - NEG_WEIGHT * self.neg_count\n        else:\n            self.score = len(self.use_true_pub_set) - NEG_WEIGHT * self.neg_count","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:30.441978Z","iopub.execute_input":"2024-07-24T12:42:30.44261Z","iopub.status.idle":"2024-07-24T12:42:30.450394Z","shell.execute_reply.started":"2024-07-24T12:42:30.442579Z","shell.execute_reply":"2024-07-24T12:42:30.449259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Timer:\n    def __init__(self):\n        self.start = time.time()\n\n    def get_current_time(self):\n        return (time.time() - self.start)\n    \n# 焼きなましのための確率を計算\ndef calc_sa_p(new_score, score, T):\n    score_diff = new_score - score\n    if score_diff >= 0:\n        return 1\n    else:\n        return math.exp(score_diff / T)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:30.881451Z","iopub.execute_input":"2024-07-24T12:42:30.881864Z","iopub.status.idle":"2024-07-24T12:42:30.888375Z","shell.execute_reply.started":"2024-07-24T12:42:30.881836Z","shell.execute_reply":"2024-07-24T12:42:30.887351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ap50(preds, labels):\n    precisions = list()\n    n_label = len(labels)\n    n_found = 0\n    for e, i in enumerate(preds):\n        if i in labels:\n            n_found += 1\n        precisions.append(n_found/(e+1)) # this is the line that is probably incorrect for competition \n    return sum(precisions)/50","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:31.248598Z","iopub.execute_input":"2024-07-24T12:42:31.249018Z","iopub.status.idle":"2024-07-24T12:42:31.25461Z","shell.execute_reply.started":"2024-07-24T12:42:31.248986Z","shell.execute_reply":"2024-07-24T12:42:31.253658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 焼きなまし  \n\n\n","metadata":{}},{"cell_type":"code","source":"len(neighbors.flatten())","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:31.697006Z","iopub.execute_input":"2024-07-24T12:42:31.697866Z","iopub.status.idle":"2024-07-24T12:42:31.713642Z","shell.execute_reply.started":"2024-07-24T12:42:31.697832Z","shell.execute_reply":"2024-07-24T12:42:31.712693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_pattern_to_neg_list = []\n\nfor sample_idx, word_pattern_to_pub_set in tqdm(enumerate(word_pattern_to_pub_set_list)):\n    \n    word_pattern_to_neg_count = dict()\n    for c, pub_set in word_pattern_to_pub_set:\n        neg_pub_set = word_pattern_to_pub_set_all[c] - pub_set\n        \n        word_pattern_to_neg_count[c] = len(neg_pub_set)\n    \n    word_pattern_to_neg_list.append(word_pattern_to_neg_count)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:32.057302Z","iopub.execute_input":"2024-07-24T12:42:32.058035Z","iopub.status.idle":"2024-07-24T12:42:32.520068Z","shell.execute_reply.started":"2024-07-24T12:42:32.058003Z","shell.execute_reply":"2024-07-24T12:42:32.519077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef word_pattern_to_query(word_pattern):\n    query_rm_cpc = [s for s in word_pattern if number_to_word[s].split(':')[0] != 'cpc']\n    query_cpc = [s for s in word_pattern if number_to_word[s].split(':')[0] == 'cpc']\n    word_pattern = query_rm_cpc + query_cpc\n    \n    query = ''\n    for i, word in enumerate(word_pattern):\n        if number_to_word[word].split(':')[0] == 'cpc':\n            if i != len(word_pattern) - 1:\n                word = number_to_word[word]\n\n                # cpc:A01H6/77* これだと770~779も含まれるので、この後に対処する\n                query += f'{word}*'\n\n                # cpc:A01H6/7? を追加して70~79に制限する。\n                base = word.split('/')[0]\n                suffix = word.split('/')[1]\n                suffix = suffix[:-1] + '?'\n                query += f'{base}/{suffix}'\n            else:\n                # 最後なら、ワイルドカードを追加する必要がない\n                query += f'{number_to_word[word]}'\n        else:\n            query += f'{number_to_word[word]}-'\n    \n    # 最後のは取り除く\n    # if len(word_pattern) >= 2 and query[-1] != '?':\n    if query[-1] == '-':\n        query = query[:-1]\n    \n    query = \"(\" + query + ')'\n    return query","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:33.210115Z","iopub.execute_input":"2024-07-24T12:42:33.210471Z","iopub.status.idle":"2024-07-24T12:42:33.220278Z","shell.execute_reply.started":"2024-07-24T12:42:33.210443Z","shell.execute_reply":"2024-07-24T12:42:33.219197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_pattern_to_char_len = dict()\n\nfor word_pattern_to_pub_set in word_pattern_to_pub_set_list:\n    for word_pattern, _ in word_pattern_to_pub_set:\n        query = word_pattern_to_query(word_pattern)\n        word_pattern_to_char_len[word_pattern] = len(query) + 4","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:33.499515Z","iopub.execute_input":"2024-07-24T12:42:33.499882Z","iopub.status.idle":"2024-07-24T12:42:34.567959Z","shell.execute_reply.started":"2024-07-24T12:42:33.499854Z","shell.execute_reply":"2024-07-24T12:42:34.566909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(word_pattern_to_char_len)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:34.569866Z","iopub.execute_input":"2024-07-24T12:42:34.570221Z","iopub.status.idle":"2024-07-24T12:42:34.576203Z","shell.execute_reply.started":"2024-07-24T12:42:34.570189Z","shell.execute_reply":"2024-07-24T12:42:34.575337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = []\nfor w, c in word_pattern_to_char_len.items():\n    counts.append(c)\n\nprint(pd.DataFrame(counts).describe())\nplt.hist(counts)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:34.577298Z","iopub.execute_input":"2024-07-24T12:42:34.577608Z","iopub.status.idle":"2024-07-24T12:42:35.182813Z","shell.execute_reply.started":"2024-07-24T12:42:34.577585Z","shell.execute_reply":"2024-07-24T12:42:35.18194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_list = []\ndefault_query = 'ti:bread'\n\n# 結果の可視化用\nsuccess_count = 0\nscore_list = []\ntrue_pub_set_len_list = [0] * len(word_pattern_to_pub_set_list)\npub_set_len_list = [0] * len(word_pattern_to_pub_set_list)\nneg_count_list = [0] * len(word_pattern_to_pub_set_list)\n\n# for _, word_pattern_counter in tqdm(enumerate(word_pattern_counter_list), total=len(word_pattern_counter_list)):\nfor sample_idx, word_pattern_to_pub_set in tqdm(enumerate(word_pattern_to_pub_set_list), total=len(word_pattern_to_pub_set_list)):\n    word_pattern_to_neg_count = word_pattern_to_neg_list[sample_idx]\n    \n    true_set = set(nn_df.values[sample_idx, 1:])\n    labels = list(nn_df.values[sample_idx, 1:])\n    labels = [num_to_pub[pub] for pub in labels]\n    \n    # 検索クエリの候補が0のとき\n    if len(word_pattern_to_pub_set) == 0:\n        score_list.append(0)\n        query_list.append(default_query)\n        continue\n    \n    word_pattern_to_pub_set_dict = dict()\n    for word_pattern, pub_set in word_pattern_to_pub_set:        \n        word_pattern_to_pub_set_dict[word_pattern] = pub_set\n    \n    use_word_pattern_list = []\n    not_use_word_pattern_list = []\n    for word_pattern, _ in word_pattern_to_pub_set:\n        not_use_word_pattern_list.append(word_pattern)\n        \n    # count_query_len に2秒くらい取られているので、事前計算\n    word_pattern_to_query_len = dict()\n    for word_pattern, _ in word_pattern_to_pub_set:\n        word_pattern_to_query_len[word_pattern] = count_query_len(word_pattern)\n        \n\n    curr_state = State([], [], set(), 0, 0)\n    curr_state.score = 0\n    \n    # queryの差分更新用\n    curr_query_len = -1\n    best_query_len = -1\n    curr_char_len = -4\n    best_char_len = -4\n\n    \n    best_states = []\n    timer = Timer()\n    \n    # 焼きなましによるスコアの変化の可視化用\n    _score_list = []\n    \n    best_state = copy.deepcopy(curr_state)\n    # best_state.calc_score()\n    best_state.score = 0\n    \n    # どの近傍が使われたかをcount\n    pattern_count = [0, 0]\n    \n    # debugで時間計測用\n    time_sum = [0]\n    \n    # neg_pickedのペナルティを強めるか\n    is_hard_penalty = False\n    \n    while True:\n        curr_time = timer.get_current_time()\n       \n        # if curr_time > max_time:\n        if curr_time > max_time:\n            break\n            \n        t = curr_time / max_time\n        T = T0**(1-t) * T1**t\n        \n        p = random.random()\n        \n        if p >= 0.5:\n            act = 'add_pattern'\n        else:\n            act = 'remove_pattern'\n\n        next_state = State([], \n                           [], \n                           set(), 0, 0,\n                           is_hard_penalty)\n\n        if act == 'add_pattern':\n            N = len(not_use_word_pattern_list)\n            if N == 0:\n                continue\n            idx = random.randint(0, N-1)\n            c = not_use_word_pattern_list[idx]\n            \n            # query_len = count_query_len(c)\n            query_len = word_pattern_to_query_len[c]\n            char_len = word_pattern_to_char_len[c]\n        \n            if curr_query_len + query_len > 50 or curr_char_len + char_len > CHAR_LIMIT:\n                is_exceed_query_limit = True\n            else:\n                is_exceed_query_limit = False\n        elif act == 'remove_pattern':\n            N = len(use_word_pattern_list)\n            if N == 0:\n                continue\n            idx = random.randint(0, N-1)\n            c = use_word_pattern_list[idx]\n            \n            query_len = word_pattern_to_query_len[c]\n            char_len = word_pattern_to_char_len[c]\n            is_exceed_query_limit = False\n\n        if is_exceed_query_limit:\n            continue\n        \n        # 差分更新にして高速化 add_pattern 0.6s remove_pattern 1.4s\n        if act == 'add_pattern':\n            neg_count = curr_state.neg_count + word_pattern_to_neg_count[c]\n            use_true_pub_set = curr_state.use_true_pub_set | word_pattern_to_pub_set_dict[c]\n            \n            update_count = len(use_true_pub_set) - len(curr_state.use_true_pub_set)\n            # recall向上しないならcontinue\n            if update_count == 0:\n                continue\n        else:\n            neg_count = 0\n            use_true_pub_set = set()\n\n            for word_pattern in use_word_pattern_list:\n                if act == 'remove_pattern' and word_pattern == c:\n                    continue\n\n                neg_count += word_pattern_to_neg_count[word_pattern]\n                if len(use_true_pub_set) + neg_count > SUB_MAX_COUNT:\n                    break\n                   \n                use_true_pub_set |= word_pattern_to_pub_set_dict[word_pattern]\n            \n        \n        # 検索結果数が閾値より大きいときは候補にしない。\n        if len(use_true_pub_set) + neg_count > SUB_MAX_COUNT or neg_count > NEG_MAX_COUNT:\n            continue\n\n        # next_state.use_pub_set = use_pub_set\n        next_state.neg_count = neg_count\n        next_state.use_true_pub_set = use_true_pub_set\n        next_state.calc_score()\n        \n        # 新しい状態を受理する確率\n        sa_p = calc_sa_p(next_state.score, curr_state.score, T)\n\n        if random.random() < sa_p:\n            curr_state = next_state\n            # curr_query_list = next_query_list\n            if act == 'add_pattern':\n                c = not_use_word_pattern_list.pop(idx)\n                use_word_pattern_list.append(c)\n                curr_query_len += query_len\n                curr_char_len += char_len\n            elif act == 'remove_pattern':\n                c = use_word_pattern_list.pop(idx)\n                not_use_word_pattern_list.append(c)\n                curr_query_len -= query_len\n                curr_char_len -= char_len\n        else:\n            pass\n\n        # _score_list.append(len(curr_state.use_true_pub_set))\n        _score_list.append(curr_state.score)\n        \n        \n        if curr_state.score > best_state.score:\n            best_state = curr_state\n            best_state.use_word_pattern_list = copy.deepcopy(use_word_pattern_list)\n            # best_query_list = copy.deepcopy(curr_query_list)\n            best_query_len = curr_query_len\n            best_char_len = curr_char_len\n            \n            if best_state.score >= 35 and not is_hard_penalty:\n                is_hard_penalty = True\n                curr_state.is_hard_penalty = True\n                curr_state.calc_score()\n                best_state.is_hard_penalty = True\n                best_state.calc_score()\n        \n        \n        if act == 'add_pattern':\n            pattern_count[0] += 1\n        else:\n            pattern_count[1] += 1\n    # print(time_sum)\n    # print(pattern_count)\n    # debug用\n    if sample_idx < 50:\n        plt.plot(_score_list)\n        plt.show()\n    \n    # best_stateで提出用のqueryを作る\n    if best_state is None:\n        print('error: best_state is None')\n        print(1/0)\n        # query_list.append(default_query)\n    else:\n        \"\"\"\n        use_true_pub_set = best_state.use_true_pub_set\n        use_pub_set = best_state.use_pub_set\n        \"\"\"\n        use_true_pub_set = set()\n        use_pub_set = set()\n        for c in best_state.use_word_pattern_list:\n            use_pub_set |= word_pattern_to_pub_set_all[c]\n            use_true_pub_set |= word_pattern_to_pub_set_dict[c]\n        \n        use_word_pattern_list = best_state.use_word_pattern_list\n        \n        true_pub_set_len_list[sample_idx] = len(use_true_pub_set)\n        pub_set_len_list[sample_idx] = len(use_pub_set)\n        neg_count_list[sample_idx] = best_state.neg_count\n        \n        query = ''\n        for i, word_pattern in enumerate(use_word_pattern_list):\n            prev_query = query\n\n            _query = word_pattern_to_query(word_pattern)\n            \n            query += _query\n            \n            if i != len(use_word_pattern_list) - 1:\n                query += ' OR '\n                \n            if whoosh_utils.count_query_tokens(query) > 50 or len(query) > CHAR_LIMIT + 500:\n \n                # エラー出して止まるようにする\n                print('query limit error')\n                print(1/0)\n                \n                query = prev_query\n                break\n        \n        # assert best_query_len == len(query.split())\n            \n        query_list.append(query)\n        success_count += 1\n        \n        if IS_TRAIN:\n            results = whoosh_utils.execute_query(query, qp, searcher)\n            result_set = set(results)\n            n_pick = len(set(labels) & result_set)\n            score = ap50(results + [-1] * (50 - len(results)), labels)\n        else:\n            score = 0\n            n_pick = 0\n        \n        if sample_idx < 50:\n            print(query)\n            print('ap50', score)\n            print('n_pick', n_pick)\n            print('true_count', true_pub_set_len_list[sample_idx])\n            print('neg_count', neg_count_list[sample_idx])\n            print(len(query), best_char_len)\n        \n        score_list.append(score)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T12:42:35.185116Z","iopub.execute_input":"2024-07-24T12:42:35.185501Z","iopub.status.idle":"2024-07-24T12:46:48.468469Z","shell.execute_reply.started":"2024-07-24T12:42:35.185452Z","shell.execute_reply":"2024-07-24T12:46:48.46754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(true_pub_set_len_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.152679Z","iopub.execute_input":"2024-07-24T13:00:09.153062Z","iopub.status.idle":"2024-07-24T13:00:09.159409Z","shell.execute_reply.started":"2024-07-24T13:00:09.153034Z","shell.execute_reply":"2024-07-24T13:00:09.158542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(true_pub_set_len_list)\nplt.title('true_pub_set_len_list distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.30466Z","iopub.execute_input":"2024-07-24T13:00:09.305022Z","iopub.status.idle":"2024-07-24T13:00:09.602713Z","shell.execute_reply.started":"2024-07-24T13:00:09.304994Z","shell.execute_reply":"2024-07-24T13:00:09.60175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(score_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.604585Z","iopub.execute_input":"2024-07-24T13:00:09.605207Z","iopub.status.idle":"2024-07-24T13:00:09.611733Z","shell.execute_reply.started":"2024-07-24T13:00:09.605171Z","shell.execute_reply":"2024-07-24T13:00:09.610758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(score_list)\nplt.title('score_list distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.61311Z","iopub.execute_input":"2024-07-24T13:00:09.613395Z","iopub.status.idle":"2024-07-24T13:00:09.849671Z","shell.execute_reply.started":"2024-07-24T13:00:09.613372Z","shell.execute_reply":"2024-07-24T13:00:09.848653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(pub_set_len_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.85118Z","iopub.execute_input":"2024-07-24T13:00:09.851466Z","iopub.status.idle":"2024-07-24T13:00:09.857744Z","shell.execute_reply.started":"2024-07-24T13:00:09.851442Z","shell.execute_reply":"2024-07-24T13:00:09.856863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(pub_set_len_list)\nplt.title('pub_set_len_list distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:09.941245Z","iopub.execute_input":"2024-07-24T13:00:09.941665Z","iopub.status.idle":"2024-07-24T13:00:10.237782Z","shell.execute_reply.started":"2024-07-24T13:00:09.941634Z","shell.execute_reply":"2024-07-24T13:00:10.236889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(neg_count_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:10.239674Z","iopub.execute_input":"2024-07-24T13:00:10.240026Z","iopub.status.idle":"2024-07-24T13:00:10.247646Z","shell.execute_reply.started":"2024-07-24T13:00:10.239993Z","shell.execute_reply":"2024-07-24T13:00:10.246811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(neg_count_list)\nplt.title('neg_count_list distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:10.253502Z","iopub.execute_input":"2024-07-24T13:00:10.25378Z","iopub.status.idle":"2024-07-24T13:00:10.478669Z","shell.execute_reply.started":"2024-07-24T13:00:10.253758Z","shell.execute_reply":"2024-07-24T13:00:10.477724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pub_set_allで計算した、正しいneg_count_list","metadata":{}},{"cell_type":"code","source":"neg_count_list_2 = [pub_set_len_list[i] - true_pub_set_len_list[i]\n                  for i in range(len(pub_set_len_list))]\nnp.mean(neg_count_list_2)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:10.592621Z","iopub.execute_input":"2024-07-24T13:00:10.59325Z","iopub.status.idle":"2024-07-24T13:00:10.600274Z","shell.execute_reply.started":"2024-07-24T13:00:10.59322Z","shell.execute_reply":"2024-07-24T13:00:10.599326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(neg_count_list_2)\nplt.title('neg_count_list_2 distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:10.752055Z","iopub.execute_input":"2024-07-24T13:00:10.752703Z","iopub.status.idle":"2024-07-24T13:00:11.045629Z","shell.execute_reply.started":"2024-07-24T13:00:10.752672Z","shell.execute_reply":"2024-07-24T13:00:11.044728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_count_list_diff = [neg_count_list[i] - neg_count_list_2[i] for i in range(len(neg_count_list))]\nprint(np.mean(neg_count_list_diff ))\nplt.hist(neg_count_list_diff)\nplt.title('neg_count_list_diff distribution')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.047048Z","iopub.execute_input":"2024-07-24T13:00:11.047303Z","iopub.status.idle":"2024-07-24T13:00:11.278335Z","shell.execute_reply.started":"2024-07-24T13:00:11.047281Z","shell.execute_reply":"2024-07-24T13:00:11.277456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN:\n    [print(s) for s in true_pub_set_len_list[:50]]\nelse:\n    print(score_list[:10])","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.280336Z","iopub.execute_input":"2024-07-24T13:00:11.280959Z","iopub.status.idle":"2024-07-24T13:00:11.286751Z","shell.execute_reply.started":"2024-07-24T13:00:11.280924Z","shell.execute_reply":"2024-07-24T13:00:11.285791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAIN:\n    [print(s) for s in score_list[:50]]\nelse:\n    print(score_list[:10])","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.288056Z","iopub.execute_input":"2024-07-24T13:00:11.288374Z","iopub.status.idle":"2024-07-24T13:00:11.29645Z","shell.execute_reply.started":"2024-07-24T13:00:11.288336Z","shell.execute_reply":"2024-07-24T13:00:11.295691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_count_list = [whoosh_utils.count_query_tokens(query) for query in query_list]\nprint(max(query_count_list))\nplt.hist(query_count_list)\n\nplt.title('token len distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.414333Z","iopub.execute_input":"2024-07-24T13:00:11.415118Z","iopub.status.idle":"2024-07-24T13:00:11.705605Z","shell.execute_reply.started":"2024-07-24T13:00:11.415081Z","shell.execute_reply":"2024-07-24T13:00:11.704722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"character_len_list = [len(query) for query in query_list]\nprint(max(character_len_list))\nplt.hist(character_len_list)\nplt.title('character len distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.707502Z","iopub.execute_input":"2024-07-24T13:00:11.70794Z","iopub.status.idle":"2024-07-24T13:00:11.976879Z","shell.execute_reply.started":"2024-07-24T13:00:11.707906Z","shell.execute_reply":"2024-07-24T13:00:11.976008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(nn_df), len(query_list), success_count","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:11.978459Z","iopub.execute_input":"2024-07-24T13:00:11.978766Z","iopub.status.idle":"2024-07-24T13:00:11.984659Z","shell.execute_reply.started":"2024-07-24T13:00:11.978742Z","shell.execute_reply":"2024-07-24T13:00:11.983792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make oof","metadata":{}},{"cell_type":"code","source":"if IS_TRAIN:\n    columns = ['query_list',\n               'true_pub_set_len_list',\n               'score_list',\n               'pub_set_len_list',\n               'neg_count_list',\n               'neg_count_list_2',\n               'query_count_list',\n               'character_len_list']\n    values = np.array([ query_list,\n               true_pub_set_len_list,\n               score_list,\n               pub_set_len_list,\n               neg_count_list,\n               neg_count_list_2,\n               query_count_list,\n               character_len_list]).T\n    \n    \n    oof_df = pd.DataFrame(values, columns=columns)\n    \n    print(oof_df.head())\n    oof_df.to_csv('oof_df.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:12.097302Z","iopub.execute_input":"2024-07-24T13:00:12.098108Z","iopub.status.idle":"2024-07-24T13:00:12.372092Z","shell.execute_reply.started":"2024-07-24T13:00:12.098072Z","shell.execute_reply":"2024-07-24T13:00:12.371214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(true_pub_set_len_list, score_list)\nplt.xlabel('true_pub_set_len_list')\nplt.ylabel('score_list')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:12.37346Z","iopub.execute_input":"2024-07-24T13:00:12.373764Z","iopub.status.idle":"2024-07-24T13:00:12.61169Z","shell.execute_reply.started":"2024-07-24T13:00:12.373739Z","shell.execute_reply":"2024-07-24T13:00:12.610707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = [i for i in range(len(true_pub_set_len_list)) if score_list[i] != 0]\ntrue_pub_set_len_list = [true_pub_set_len_list[i] for i in idxs]\nscore_list = [score_list[i] for i in idxs]\nprint(len(true_pub_set_len_list))\nplt.scatter(true_pub_set_len_list, score_list)\nplt.xlabel('true_pub_set_len_list')\nplt.ylabel('score_list')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:12.613182Z","iopub.execute_input":"2024-07-24T13:00:12.613473Z","iopub.status.idle":"2024-07-24T13:00:12.907812Z","shell.execute_reply.started":"2024-07-24T13:00:12.613448Z","shell.execute_reply":"2024-07-24T13:00:12.906878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"markdown","source":"# 'id:'がqueryに含まれる場合は消す","metadata":{}},{"cell_type":"code","source":"default_query = 'ti:bread'\n\nfor i in range(len(query_list)):\n    if 'id:' in query_list[i]:\n        query_list[i] = default_query\n\n\"\"\"\nfor i in range(len(query_list)):\n    try:\n        result = whoosh_utils.execute_query(query_list[i], qp, searcher)\n    except Exception as e:\n        query_list[i] = default_query\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:12.947903Z","iopub.execute_input":"2024-07-24T13:00:12.948199Z","iopub.status.idle":"2024-07-24T13:00:12.956167Z","shell.execute_reply.started":"2024-07-24T13:00:12.948175Z","shell.execute_reply":"2024-07-24T13:00:12.95523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# クエリの実行エラーを解消","metadata":{}},{"cell_type":"code","source":"for i in range(len(query_list)):\n    if query_list[i] == '':\n        query_list[i] = default_query","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:13.305146Z","iopub.execute_input":"2024-07-24T13:00:13.305803Z","iopub.status.idle":"2024-07-24T13:00:13.310544Z","shell.execute_reply.started":"2024-07-24T13:00:13.305772Z","shell.execute_reply":"2024-07-24T13:00:13.309541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nif not IS_TRAIN:\n    # train_idx = whoosh_utils.load_index('/kaggle/input/create-valid-index-add-claim/test_index')\n    train_idx = whoosh_utils.load_index('/kaggle/input/uspto-test-index/test_index')\n    searcher = whoosh_utils.get_searcher(train_idx)\n    qp = whoosh_utils.get_query_parser()\n\n    # query = 'ti:balloons OR ti:string'\n    query = '(cpc:A01H6/42 ab:cuttings) OR (ab:regal-ab:rooting) OR (ab:magenta-ab:umbels) OR (ti:geranium-ti:variety-ab:cherry-ab:foliage-ab:pink-clm:geranium) OR (ti:bravo-ab:bicolored) OR (ab:foliage-ab:geranium-ab:regal-clm:geranium-detd:duchess) OR (ti:debutante-ti:pelargonium) OR (ti:geranium-ab:bright-ab:garden-ab:geranium-ab:mounded-ab:zonal-clm:geranium-clm:oglger14007-clm:zonal-detd:oglger14007) OR (ti:geranium-ti:variety-ab:blossoms-ab:foliage-ab:orchid-clm:geranium) OR (ti:louise-ab:ivy) OR (ti:hots) OR (ti:geranium-ti:variety-ab:orange-ab:rain-clm:geranium)'\n    print(whoosh_utils.execute_query(query, qp, searcher)[:5])\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:13.497305Z","iopub.execute_input":"2024-07-24T13:00:13.498138Z","iopub.status.idle":"2024-07-24T13:00:13.504577Z","shell.execute_reply.started":"2024-07-24T13:00:13.498106Z","shell.execute_reply":"2024-07-24T13:00:13.503682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nfor i in range(len(query_list)):\n    try:\n        result = whoosh_utils.execute_query(query_list[i], qp, searcher)\n    except Exception as e:\n        print(e)\n        query_list[i] = default_query\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:13.665624Z","iopub.execute_input":"2024-07-24T13:00:13.666345Z","iopub.status.idle":"2024-07-24T13:00:13.67205Z","shell.execute_reply.started":"2024-07-24T13:00:13.666314Z","shell.execute_reply":"2024-07-24T13:00:13.671048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_TRAIN:\n    sub = pd.read_csv('/kaggle/input/uspto-explainable-ai/sample_submission.csv')\n    \n    sub['query'] = query_list\n    \n    sub.to_csv('submission.csv', index=False)\n    \n    print(sub)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T13:00:13.851133Z","iopub.execute_input":"2024-07-24T13:00:13.851512Z","iopub.status.idle":"2024-07-24T13:00:13.856553Z","shell.execute_reply.started":"2024-07-24T13:00:13.851463Z","shell.execute_reply":"2024-07-24T13:00:13.855624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}