{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install elasticsearch[async] pandasticsearch","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:31.564889Z","iopub.execute_input":"2021-11-22T02:19:31.565416Z","iopub.status.idle":"2021-11-22T02:19:38.800313Z","shell.execute_reply.started":"2021-11-22T02:19:31.565370Z","shell.execute_reply":"2021-11-22T02:19:38.798999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nfrom kaggle_secrets import UserSecretsClient\nfrom elasticsearch import Elasticsearch, helpers\nfrom pandasticsearch import Select","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:38.802036Z","iopub.execute_input":"2021-11-22T02:19:38.802322Z","iopub.status.idle":"2021-11-22T02:19:38.807067Z","shell.execute_reply.started":"2021-11-22T02:19:38.802287Z","shell.execute_reply":"2021-11-22T02:19:38.806198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\n\nELASTICSEARCH_URL = user_secrets.get_secret('ELASTICSEARCH_URL')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:38.808402Z","iopub.execute_input":"2021-11-22T02:19:38.808628Z","iopub.status.idle":"2021-11-22T02:19:38.960755Z","shell.execute_reply.started":"2021-11-22T02:19:38.808601Z","shell.execute_reply":"2021-11-22T02:19:38.959775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df = pd.read_csv('/kaggle/input/trec-covid-information-retrieval/CORD-19/CORD-19/metadata.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:38.963467Z","iopub.execute_input":"2021-11-22T02:19:38.963820Z","iopub.status.idle":"2021-11-22T02:19:41.941092Z","shell.execute_reply.started":"2021-11-22T02:19:38.963775Z","shell.execute_reply":"2021-11-22T02:19:41.940052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:41.942646Z","iopub.execute_input":"2021-11-22T02:19:41.943016Z","iopub.status.idle":"2021-11-22T02:19:41.967053Z","shell.execute_reply.started":"2021-11-22T02:19:41.942946Z","shell.execute_reply":"2021-11-22T02:19:41.965932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:41.968584Z","iopub.execute_input":"2021-11-22T02:19:41.968898Z","iopub.status.idle":"2021-11-22T02:19:42.095760Z","shell.execute_reply.started":"2021-11-22T02:19:41.968858Z","shell.execute_reply":"2021-11-22T02:19:42.094682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_records = metadata_df[['cord_uid', 'title', 'abstract']].fillna('').to_dict(orient='records')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.097292Z","iopub.execute_input":"2021-11-22T02:19:42.097516Z","iopub.status.idle":"2021-11-22T02:19:42.683883Z","shell.execute_reply.started":"2021-11-22T02:19:42.097490Z","shell.execute_reply":"2021-11-22T02:19:42.683113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = Elasticsearch(ELASTICSEARCH_URL)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.685258Z","iopub.execute_input":"2021-11-22T02:19:42.685481Z","iopub.status.idle":"2021-11-22T02:19:42.691895Z","shell.execute_reply.started":"2021-11-22T02:19:42.685453Z","shell.execute_reply":"2021-11-22T02:19:42.690975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# es.indices.delete(index='trec_covid', ignore=[400, 404])\n# es.indices.create(index='trec_covid')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.693177Z","iopub.execute_input":"2021-11-22T02:19:42.693478Z","iopub.status.idle":"2021-11-22T02:19:42.700941Z","shell.execute_reply.started":"2021-11-22T02:19:42.693437Z","shell.execute_reply":"2021-11-22T02:19:42.699971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helpers.bulk(es, metadata_records, index='trec_covid', doc_type='metadata')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.702271Z","iopub.execute_input":"2021-11-22T02:19:42.702570Z","iopub.status.idle":"2021-11-22T02:19:42.713602Z","shell.execute_reply.started":"2021-11-22T02:19:42.702528Z","shell.execute_reply":"2021-11-22T02:19:42.712828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Select.from_dict(es.search(index='trec_covid')).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.714822Z","iopub.execute_input":"2021-11-22T02:19:42.715456Z","iopub.status.idle":"2021-11-22T02:19:42.836698Z","shell.execute_reply.started":"2021-11-22T02:19:42.715407Z","shell.execute_reply":"2021-11-22T02:19:42.836029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics_rnd3_df = pd.read_csv('/kaggle/input/trec-covid-information-retrieval/topics-rnd3.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.837775Z","iopub.execute_input":"2021-11-22T02:19:42.838482Z","iopub.status.idle":"2021-11-22T02:19:42.844684Z","shell.execute_reply.started":"2021-11-22T02:19:42.838446Z","shell.execute_reply":"2021-11-22T02:19:42.844014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics_rnd3_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.845843Z","iopub.execute_input":"2021-11-22T02:19:42.846684Z","iopub.status.idle":"2021-11-22T02:19:42.861550Z","shell.execute_reply.started":"2021-11-22T02:19:42.846643Z","shell.execute_reply":"2021-11-22T02:19:42.860508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics_rnd3_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.862924Z","iopub.execute_input":"2021-11-22T02:19:42.863393Z","iopub.status.idle":"2021-11-22T02:19:42.875020Z","shell.execute_reply.started":"2021-11-22T02:19:42.863358Z","shell.execute_reply":"2021-11-22T02:19:42.873842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = []\nfor _, row in topics_rnd3_df.iterrows():\n    result_search = es.search(index='trec_covid', query={\n        'multi_match': {\n            'query': row['query'],\n            'fields': ['title', 'abstract']\n        }\n    })\n    result_search_df = Select.from_dict(result_search).to_pandas()\n    submission.extend((row['topic-id'], cord_uid) for cord_uid in result_search_df['cord_uid'])\nsubmission_df = pd.DataFrame(submission, columns=('topic-id', 'cord-id'))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:42.877043Z","iopub.execute_input":"2021-11-22T02:19:42.877515Z","iopub.status.idle":"2021-11-22T02:19:43.909805Z","shell.execute_reply.started":"2021-11-22T02:19:42.877477Z","shell.execute_reply":"2021-11-22T02:19:43.908877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:43.911448Z","iopub.execute_input":"2021-11-22T02:19:43.911767Z","iopub.status.idle":"2021-11-22T02:19:43.921907Z","shell.execute_reply.started":"2021-11-22T02:19:43.911726Z","shell.execute_reply":"2021-11-22T02:19:43.921242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:43.923214Z","iopub.execute_input":"2021-11-22T02:19:43.923664Z","iopub.status.idle":"2021-11-22T02:19:43.943425Z","shell.execute_reply.started":"2021-11-22T02:19:43.923629Z","shell.execute_reply":"2021-11-22T02:19:43.942303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T02:19:43.945903Z","iopub.execute_input":"2021-11-22T02:19:43.946629Z","iopub.status.idle":"2021-11-22T02:19:43.954535Z","shell.execute_reply.started":"2021-11-22T02:19:43.946578Z","shell.execute_reply":"2021-11-22T02:19:43.953419Z"},"trusted":true},"execution_count":null,"outputs":[]}]}