{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install hashedindex\n!apt-get update\n!apt-get install tesseract-ocr\n!apt-get install tesseract-ocr-por\n!pip install tika","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-20T04:10:23.110688Z","iopub.execute_input":"2021-11-20T04:10:23.111475Z","iopub.status.idle":"2021-11-20T04:11:00.60886Z","shell.execute_reply.started":"2021-11-20T04:10:23.111357Z","shell.execute_reply":"2021-11-20T04:11:00.607852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport hashedindex\nimport tika\nimport requests\nfrom tika import parser\nfrom hashedindex import textparser\n\n# Start running the tika service\ntika.initVM()\n\nimport os\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:13.888849Z","iopub.execute_input":"2021-11-19T23:33:13.88916Z","iopub.status.idle":"2021-11-19T23:33:13.895779Z","shell.execute_reply.started":"2021-11-19T23:33:13.889124Z","shell.execute_reply":"2021-11-19T23:33:13.894655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for dirname, _, filenames in os.walk('/kaggle/input/trec-covid-information-retrieval/CORD-19/CORD-19/document_parses/pdf_json'):\n#    for filename in filenames[:5]:\n#        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-11-19T15:08:10.014056Z","iopub.execute_input":"2021-11-19T15:08:10.014372Z","iopub.status.idle":"2021-11-19T15:08:10.023939Z","shell.execute_reply.started":"2021-11-19T15:08:10.014313Z","shell.execute_reply":"2021-11-19T15:08:10.023232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docids = pd.read_csv(\"../input/trec-covid-information-retrieval/docids-rnd3.txt\", sep = \" \", header = None)\n\ntopicColList = [\"topic-id\", \"query\"]\nmetadataColList = [\"cord_uid\", \"pdf_json_files\"]\n\ntopics = pd.read_csv(\"../input/trec-covid-information-retrieval/topics-rnd3.csv\", usecols = topicColList)\nmetadata = pd.read_csv(\"../input/trec-covid-information-retrieval/CORD-19/CORD-19/metadata.csv\", usecols = metadataColList)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:19.41084Z","iopub.execute_input":"2021-11-19T23:33:19.411142Z","iopub.status.idle":"2021-11-19T23:33:22.73504Z","shell.execute_reply.started":"2021-11-19T23:33:19.411109Z","shell.execute_reply":"2021-11-19T23:33:22.734075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:26.187999Z","iopub.execute_input":"2021-11-19T23:33:26.188298Z","iopub.status.idle":"2021-11-19T23:33:26.200551Z","shell.execute_reply.started":"2021-11-19T23:33:26.188264Z","shell.execute_reply":"2021-11-19T23:33:26.199347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docids.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:29.259315Z","iopub.execute_input":"2021-11-19T23:33:29.259972Z","iopub.status.idle":"2021-11-19T23:33:29.275285Z","shell.execute_reply.started":"2021-11-19T23:33:29.259901Z","shell.execute_reply":"2021-11-19T23:33:29.273957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:32.691264Z","iopub.execute_input":"2021-11-19T23:33:32.691552Z","iopub.status.idle":"2021-11-19T23:33:32.703122Z","shell.execute_reply.started":"2021-11-19T23:33:32.691519Z","shell.execute_reply":"2021-11-19T23:33:32.701646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = hashedindex.HashedIndex()","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:33:37.974944Z","iopub.execute_input":"2021-11-19T23:33:37.975282Z","iopub.status.idle":"2021-11-19T23:33:37.980056Z","shell.execute_reply.started":"2021-11-19T23:33:37.975238Z","shell.execute_reply":"2021-11-19T23:33:37.979109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadataSize = len(metadata)\ntopicsSize = len(topics)\n\n#print(\"Metadata Size:\", metadataSize) pegar 15 mil\nprint(\"Topics Size:\", topicsSize)\nmetadataSize = 20000\n\ntotalRuns = metadataSize * topicsSize\n\nfrom IPython.display import clear_output","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:34:14.279914Z","iopub.execute_input":"2021-11-19T23:34:14.280486Z","iopub.status.idle":"2021-11-19T23:34:14.287651Z","shell.execute_reply.started":"2021-11-19T23:34:14.28045Z","shell.execute_reply":"2021-11-19T23:34:14.286402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\niteration = 1\nfor index_t, topic in topics.iterrows():\n    for index_m, data in metadata.head(metadataSize).iterrows():\n        progress = (iteration / totalRuns) * 100 \n        clear_output(wait=True)\n        print(\"Progress:\", progress, \"(%)\")\n        if isinstance(data[\"pdf_json_files\"], str):\n            try:\n                text = \"\"\n                filecontent = parser.from_file(\"/kaggle/input/trec-covid-information-retrieval/CORD-19/CORD-19/\" + data[\"pdf_json_files\"])\n                for line in filecontent[\"content\"].splitlines():\n                    if \"\\\"text\\\"\" in line or \"\\\"title\\\"\" in line:\n                        text += line\n                \n                text = text.replace(\"\\\"text\\\":\", \"\").replace(\"\\\"title\\\":\", \"\").replace(\"\\\"\", \"\")\n                \n                occurencesCount = text.count(topic[\"query\"])\n                \n                results.append([topic[\"topic-id\"], data[\"cord_uid\"], occurencesCount]) \n            except:\n                continue\n        iteration += 1","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:37:46.073728Z","iopub.execute_input":"2021-11-19T23:37:46.074039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results","metadata":{"execution":{"iopub.status.busy":"2021-11-19T23:28:52.465945Z","iopub.execute_input":"2021-11-19T23:28:52.466368Z","iopub.status.idle":"2021-11-19T23:28:52.474419Z","shell.execute_reply.started":"2021-11-19T23:28:52.466335Z","shell.execute_reply":"2021-11-19T23:28:52.472923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resultDF = pd.DataFrame(results, columns = ['topic-id', 'cord-id', 'countOccur'])\nresultDF = resultDF.sort_values(by = [\"topic-id\", \"countOccur\"], ascending=[True, False])\n\nresultDF.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T01:50:01.274721Z","iopub.execute_input":"2021-11-20T01:50:01.275011Z","iopub.status.idle":"2021-11-20T01:50:01.351998Z","shell.execute_reply.started":"2021-11-20T01:50:01.274931Z","shell.execute_reply":"2021-11-20T01:50:01.350804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = 'submission.csv'\n\nresultDF[[\"topic-id\",\"cord-id\"]].to_csv(filename,index=False)\n\nprint('Saved file: ' + filename)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T21:58:16.788211Z","iopub.execute_input":"2021-11-19T21:58:16.78865Z","iopub.status.idle":"2021-11-19T21:58:16.832597Z","shell.execute_reply.started":"2021-11-19T21:58:16.788616Z","shell.execute_reply":"2021-11-19T21:58:16.831452Z"}},"execution_count":null,"outputs":[]}]}