{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install opensearch-py","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:34:59.788811Z","iopub.execute_input":"2021-10-12T07:34:59.789206Z","iopub.status.idle":"2021-10-12T07:35:08.857691Z","shell.execute_reply.started":"2021-10-12T07:34:59.789170Z","shell.execute_reply":"2021-10-12T07:35:08.856506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# from elasticsearch import Elasticsearch\nimport numpy as np\nimport pandas as pd\nimport subprocess\nfrom subprocess import Popen, PIPE, STDOUT\nfrom opensearchpy import OpenSearch","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:35:08.859602Z","iopub.execute_input":"2021-10-12T07:35:08.859953Z","iopub.status.idle":"2021-10-12T07:35:09.108271Z","shell.execute_reply.started":"2021-10-12T07:35:08.859916Z","shell.execute_reply":"2021-10-12T07:35:09.107116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n\n# Download and extract elasticsearch\n!wget https://d3g5vo6xdbdb9a.cloudfront.net/tarball/opendistro-elasticsearch/opendistroforelasticsearch-1.13.2-linux-x64.tar.gz\n!tar -zxf opendistroforelasticsearch-1.13.2-linux-x64.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:35:09.110212Z","iopub.execute_input":"2021-10-12T07:35:09.110537Z","iopub.status.idle":"2021-10-12T07:35:31.479374Z","shell.execute_reply.started":"2021-10-12T07:35:09.110504Z","shell.execute_reply":"2021-10-12T07:35:31.478104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!chown -R daemon:daemon ./opendistroforelasticsearch-1.13.2","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:35:31.481416Z","iopub.execute_input":"2021-10-12T07:35:31.481863Z","iopub.status.idle":"2021-10-12T07:35:32.269525Z","shell.execute_reply.started":"2021-10-12T07:35:31.481808Z","shell.execute_reply":"2021-10-12T07:35:32.268289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"server = Popen(['./opendistroforelasticsearch-1.13.2/opendistro-tar-install.sh'], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1))\n!sleep 30","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:35:32.270977Z","iopub.execute_input":"2021-10-12T07:35:32.271283Z","iopub.status.idle":"2021-10-12T07:36:03.155278Z","shell.execute_reply.started":"2021-10-12T07:35:32.271253Z","shell.execute_reply":"2021-10-12T07:36:03.153924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncurl -X GET \"https://localhost:9200\" -u 'admin:admin' --insecure","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:36:03.157226Z","iopub.execute_input":"2021-10-12T07:36:03.157663Z","iopub.status.idle":"2021-10-12T07:36:03.798993Z","shell.execute_reply.started":"2021-10-12T07:36:03.157609Z","shell.execute_reply":"2021-10-12T07:36:03.797583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncurl -X GET \"https://localhost:9200/_cat/plugins?v\" -u 'admin:admin' --insecure","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:36:03.800672Z","iopub.execute_input":"2021-10-12T07:36:03.800986Z","iopub.status.idle":"2021-10-12T07:36:03.957268Z","shell.execute_reply.started":"2021-10-12T07:36:03.800952Z","shell.execute_reply":"2021-10-12T07:36:03.955902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"host = 'localhost'\nport = 9200\nauth = ('admin', 'admin') \nclient = OpenSearch(\n    hosts = [{'host': host, 'port': port}],\n    http_auth = auth,\n    use_ssl = True,\n    verify_certs = False,\n    ssl_assert_hostname = False,\n    ssl_show_warn = False\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:36:03.960253Z","iopub.execute_input":"2021-10-12T07:36:03.960642Z","iopub.status.idle":"2021-10-12T07:36:03.967890Z","shell.execute_reply.started":"2021-10-12T07:36:03.960602Z","shell.execute_reply":"2021-10-12T07:36:03.966987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_name = 'test'\nindex_body = {\n  'settings': {\n    'index': {\n      'number_of_shards': 4\n    }\n  }\n}","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:36:03.969555Z","iopub.execute_input":"2021-10-12T07:36:03.970204Z","iopub.status.idle":"2021-10-12T07:36:03.980000Z","shell.execute_reply.started":"2021-10-12T07:36:03.970156Z","shell.execute_reply":"2021-10-12T07:36:03.978932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"response = client.indices.create(index_name, body=index_body)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T07:36:03.981157Z","iopub.execute_input":"2021-10-12T07:36:03.981564Z","iopub.status.idle":"2021-10-12T07:36:04.576723Z","shell.execute_reply.started":"2021-10-12T07:36:03.981535Z","shell.execute_reply":"2021-10-12T07:36:04.575418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nCreating index:')\nprint(response)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T03:53:42.979528Z","iopub.execute_input":"2021-10-12T03:53:42.979879Z","iopub.status.idle":"2021-10-12T03:53:42.985905Z","shell.execute_reply.started":"2021-10-12T03:53:42.979851Z","shell.execute_reply":"2021-10-12T03:53:42.984748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"document = {\n  'title': 'Moneyball',\n  'director': 'Bennett Miller',\n  'year': '2011'\n}\nid = '1'\n\nresponse = client.index(\n    index = index_name,\n    body = document,\n    id = id,\n    refresh = True\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T04:02:26.720285Z","iopub.execute_input":"2021-10-12T04:02:26.721130Z","iopub.status.idle":"2021-10-12T04:02:26.850837Z","shell.execute_reply.started":"2021-10-12T04:02:26.721080Z","shell.execute_reply":"2021-10-12T04:02:26.849598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nAdding document:')\nprint(response)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T04:02:47.276650Z","iopub.execute_input":"2021-10-12T04:02:47.277019Z","iopub.status.idle":"2021-10-12T04:02:47.281903Z","shell.execute_reply.started":"2021-10-12T04:02:47.276982Z","shell.execute_reply":"2021-10-12T04:02:47.280743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q = 'miller'\nquery = {\n  'size': 5,\n  'query': {\n    'multi_match': {\n      'query': q,\n      'fields': ['title^2', 'director']\n    }\n  }\n}\n\nresponse = client.search(\n    body = query,\n    index = index_name\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T04:04:23.221176Z","iopub.execute_input":"2021-10-12T04:04:23.221596Z","iopub.status.idle":"2021-10-12T04:04:23.330491Z","shell.execute_reply.started":"2021-10-12T04:04:23.221560Z","shell.execute_reply":"2021-10-12T04:04:23.329631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nSearch results:')\nprint(response)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T04:05:13.229750Z","iopub.execute_input":"2021-10-12T04:05:13.230168Z","iopub.status.idle":"2021-10-12T04:05:13.236028Z","shell.execute_reply.started":"2021-10-12T04:05:13.230137Z","shell.execute_reply":"2021-10-12T04:05:13.235276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data = pd.read_csv(\"../input/treccovid-round-1/topic_set.csv\")\n# data.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:14:28.657554Z","iopub.execute_input":"2021-10-05T01:14:28.657832Z","iopub.status.idle":"2021-10-05T01:14:28.707111Z","shell.execute_reply.started":"2021-10-05T01:14:28.657807Z","shell.execute_reply":"2021-10-05T01:14:28.706199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_name = \"bert-base-nli-mean-tokens\"\n# model = SentenceTransformer(model_name)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:14:30.810147Z","iopub.execute_input":"2021-10-05T01:14:30.810475Z","iopub.status.idle":"2021-10-05T01:14:54.598713Z","shell.execute_reply.started":"2021-10-05T01:14:30.810437Z","shell.execute_reply":"2021-10-05T01:14:54.597264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data['batch'] = data['narrative'].apply(lambda t: model.encode([t])[0].tolist())","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:14:54.600744Z","iopub.execute_input":"2021-10-05T01:14:54.601098Z","iopub.status.idle":"2021-10-05T01:14:59.339278Z","shell.execute_reply.started":"2021-10-05T01:14:54.601062Z","shell.execute_reply":"2021-10-05T01:14:59.338595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def prepare_es_data(index, df):\n#     records = df.to_dict(orient=\"records\")\n#     es_data = []\n#     for idx, record in enumerate(records):\n#         meta_dict = {\n#               \"index\": {\n#                   \"_index\": index, \n#                   \"_id\": idx\n#               }\n#           }\n#         es_data.append(meta_dict)\n#         es_data.append(record)\n#     return es_data","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:15:53.32458Z","iopub.execute_input":"2021-10-05T01:15:53.32496Z","iopub.status.idle":"2021-10-05T01:15:53.332039Z","shell.execute_reply.started":"2021-10-05T01:15:53.324923Z","shell.execute_reply":"2021-10-05T01:15:53.330011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def index_es_data(index, es_data):\n#     res = es_client.bulk(index=index, body=es_data, refresh = True)\n#     print(\"Errors: {}, Num of records indexed: {}\".format(res[\"errors\"], len(res[\"items\"])))","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:02.107389Z","iopub.execute_input":"2021-10-05T01:16:02.107693Z","iopub.status.idle":"2021-10-05T01:16:02.112004Z","shell.execute_reply.started":"2021-10-05T01:16:02.10767Z","shell.execute_reply":"2021-10-05T01:16:02.111242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# es_data = prepare_es_data(index=\"sentence\",  df=data)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:30.276726Z","iopub.execute_input":"2021-10-05T01:16:30.277042Z","iopub.status.idle":"2021-10-05T01:16:30.282406Z","shell.execute_reply.started":"2021-10-05T01:16:30.277016Z","shell.execute_reply":"2021-10-05T01:16:30.280987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# index_es_data(index=\"sentence\", es_data=es_data)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:36.892364Z","iopub.execute_input":"2021-10-05T01:16:36.892731Z","iopub.status.idle":"2021-10-05T01:16:37.335812Z","shell.execute_reply.started":"2021-10-05T01:16:36.8927Z","shell.execute_reply":"2021-10-05T01:16:37.334867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vecquery = model.encode([\"coronavirus social distancing impact\"])[0]","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:49.221684Z","iopub.execute_input":"2021-10-05T01:16:49.22206Z","iopub.status.idle":"2021-10-05T01:16:49.356197Z","shell.execute_reply.started":"2021-10-05T01:16:49.222027Z","shell.execute_reply":"2021-10-05T01:16:49.355239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# query = {\n#   \"size\": 6,\n#   \"query\": {\n#     \"script_score\": {\n#       \"query\" : {\n#         \"match_all\": {}\n#       },\n#       \"script\": {\n#         \"source\": \"1 / (1 + l2norm(params.queryVector, 'batch'))\",\n#         \"params\": {\n#           \"queryVector\": vecquery\n#         }\n#       }\n#     }\n#   },\n#   \"sort\": { \n#       \"_score\": { \"order\": \"desc\" } \n#   }\n# }\n\n# results = es_client.search(index=\"sentence\", body=query)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:52.468495Z","iopub.execute_input":"2021-10-05T01:16:52.468835Z","iopub.status.idle":"2021-10-05T01:16:52.645923Z","shell.execute_reply.started":"2021-10-05T01:16:52.468802Z","shell.execute_reply":"2021-10-05T01:16:52.645156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for hit in results[\"hits\"][\"hits\"]:\n#     print([hit[\"_source\"][\"query\"],\"---\", hit[\"_score\"]])","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:16:54.971731Z","iopub.execute_input":"2021-10-05T01:16:54.973076Z","iopub.status.idle":"2021-10-05T01:16:54.979292Z","shell.execute_reply.started":"2021-10-05T01:16:54.973015Z","shell.execute_reply":"2021-10-05T01:16:54.977956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0.06816692 - 0.06667824","metadata":{"execution":{"iopub.status.busy":"2021-10-05T01:17:20.133123Z","iopub.execute_input":"2021-10-05T01:17:20.133432Z","iopub.status.idle":"2021-10-05T01:17:20.139779Z","shell.execute_reply.started":"2021-10-05T01:17:20.133405Z","shell.execute_reply":"2021-10-05T01:17:20.138319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}