{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport spacy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics = pd.read_csv('../input/trec-covid-information-retrieval/topics-rnd3.csv')\ndocuments_metadata = pd.read_csv('../input/trec-covid-information-retrieval/CORD-19/CORD-19/metadata.csv')\ndocids = pd.read_csv('../input/trec-covid-information-retrieval/docids-rnd3.txt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\nimport en_core_web_sm\nnlp = en_core_web_sm.load()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clear_df(text):\n    text = text.apply(lambda x: [token for token in x if token not in string.punctuation and not token.isnumeric()])\n    text = text.apply(lambda x: ''.join(x))\n    text = text.apply(lambda x: [token.text.lower() for token in nlp(x) if (token.is_stop == False and len(token.text)>3)])\n    text = text.apply(lambda x: ' '.join(x))\n    text = text.apply(lambda x: [token.lemma_ for token in nlp(x)])\n    text = text.apply(lambda x: ' '.join(x))\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics['query'] = clear_df(topics['query'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qrels = pd.read_csv('../input/trec-covid-information-retrieval/qrels.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts_for_search = pd.DataFrame()\ntexts_for_search['cord_uid'] = documents_metadata['cord_uid']\ntexts_for_search['texts'] = documents_metadata['title'] + documents_metadata['abstract']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents = pd.merge(qrels, docids, left_on='cord-id',right_on='000ajevz', how=\"outer\", indicator=True)\nvalid_documents_ids = valid_documents[valid_documents['_merge'] == 'right_only']['000ajevz']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts_for_search = texts_for_search[texts_for_search['texts'].notna()]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data = pd.merge(valid_documents_ids,documents_metadata,left_on='000ajevz',right_on='cord_uid',how='right', indicator=True)\nvalid_documents_data[valid_documents_data['_merge'] != 'right_only']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear = valid_documents_data[valid_documents_data['_merge'] != 'right_only']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"searchfor = ['SARS-CoV-2', 'coronavirus', 'COVID', 'COVID-19']\nvalid_documents_data_clear_filtered = valid_documents_data_clear[valid_documents_data_clear.title.str.contains('|'.join(searchfor), na=False)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear_filtered.drop(['000ajevz', 'sha', 'publish_time', 'source_x', 'doi', 'pmcid', 'pubmed_id', 'license', 'authors', 'journal', 'mag_id', 'who_covidence_id', 'arxiv_id', 'pdf_json_files', 'pmc_json_files', 'url', 's2_id', '_merge'], axis=1, inplace=True)\nvalid_documents_data_clear_filtered","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear_filtered['text'] = valid_documents_data_clear_filtered['title'] + valid_documents_data_clear_filtered['abstract']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear_filtered.drop(['title', 'abstract'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear_filtered = valid_documents_data_clear_filtered.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install hashedindex\nimport hashedindex","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = hashedindex.HashedIndex()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_documents_data_clear_filtered","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ind, row_docs in valid_documents_data_clear_filtered.iterrows():\n    for ind, row_topics in topics.iterrows():\n        for word in row_topics['query'].split():\n            if word in row_docs['text']:\n                index.add_term_occurrence(row_topics['query'], str(row_docs['cord_uid']))\n            ","metadata":{"execution":{"iopub.status.busy":"2021-11-21T20:53:18.062746Z","iopub.execute_input":"2021-11-21T20:53:18.063051Z","iopub.status.idle":"2021-11-21T20:54:01.130193Z","shell.execute_reply.started":"2021-11-21T20:53:18.063021Z","shell.execute_reply":"2021-11-21T20:54:01.129310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index.items().keys()\nindex.get_documents('coronavirus response weather change').most_common(10)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T20:56:18.885655Z","iopub.execute_input":"2021-11-21T20:56:18.885923Z","iopub.status.idle":"2021-11-21T20:56:18.894974Z","shell.execute_reply.started":"2021-11-21T20:56:18.885895Z","shell.execute_reply":"2021-11-21T20:56:18.894017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.DataFrame(columns=['topic-id', 'cord-id'])\nfor ind, row_topics in topics.iterrows():\n    for cord_id, occ in index.get_documents(row_topics['query']).most_common(10):\n        results.loc[len(results)] = [row_topics['topic-id'],cord_id]\n","metadata":{"execution":{"iopub.status.busy":"2021-11-21T21:28:03.597274Z","iopub.execute_input":"2021-11-21T21:28:03.597740Z","iopub.status.idle":"2021-11-21T21:28:04.520845Z","shell.execute_reply.started":"2021-11-21T21:28:03.597707Z","shell.execute_reply":"2021-11-21T21:28:04.520209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T21:34:53.520363Z","iopub.execute_input":"2021-11-21T21:34:53.520691Z","iopub.status.idle":"2021-11-21T21:34:53.527775Z","shell.execute_reply.started":"2021-11-21T21:34:53.520657Z","shell.execute_reply":"2021-11-21T21:34:53.527003Z"},"trusted":true},"execution_count":null,"outputs":[]}]}