{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport warnings\nimport transformers","metadata":{"papermill":{"duration":0.27366,"end_time":"2022-08-11T17:36:32.711340","exception":false,"start_time":"2022-08-11T17:36:32.437680","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:15.249893Z","iopub.execute_input":"2022-08-11T20:32:15.251103Z","iopub.status.idle":"2022-08-11T20:32:15.520918Z","shell.execute_reply.started":"2022-08-11T20:32:15.250959Z","shell.execute_reply":"2022-08-11T20:32:15.519875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings('ignore')\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nos.environ['TOKENIZERS_PARALLELISM'] = 'true'\nsys.path.insert(0, '../input/ai4code-source')\ntransformers.utils.logging.set_verbosity_error()","metadata":{"papermill":{"duration":0.011958,"end_time":"2022-08-11T17:36:32.726813","exception":false,"start_time":"2022-08-11T17:36:32.714855","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:15.523367Z","iopub.execute_input":"2022-08-11T20:32:15.524067Z","iopub.status.idle":"2022-08-11T20:32:15.530692Z","shell.execute_reply.started":"2022-08-11T20:32:15.524025Z","shell.execute_reply":"2022-08-11T20:32:15.529362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom transformers import AutoTokenizer","metadata":{"papermill":{"duration":4.634901,"end_time":"2022-08-11T17:36:37.364241","exception":false,"start_time":"2022-08-11T17:36:32.729340","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:15.532295Z","iopub.execute_input":"2022-08-11T20:32:15.532836Z","iopub.status.idle":"2022-08-11T20:32:20.478488Z","shell.execute_reply.started":"2022-08-11T20:32:15.532799Z","shell.execute_reply":"2022-08-11T20:32:20.477451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from utils import submit\nfrom dataset import get_dataset\nfrom model import get_reg_model\nfrom loader import get_pct_ranks\nfrom loader import load_notebooks\nfrom model import get_match_model\nfrom dataset import get_reg_input\nfrom extractor import extract_reg_data\nfrom extractor import extract_match_data\nfrom model import pairwise_cosine_similarity","metadata":{"papermill":{"duration":1.588491,"end_time":"2022-08-11T17:36:38.956227","exception":false,"start_time":"2022-08-11T17:36:37.367736","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:20.481262Z","iopub.execute_input":"2022-08-11T20:32:20.481965Z","iopub.status.idle":"2022-08-11T20:32:21.904618Z","shell.execute_reply.started":"2022-08-11T20:32:20.481903Z","shell.execute_reply":"2022-08-11T20:32:21.903537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = load_notebooks('../input/AI4Code/test', 150000)\ntest_df['pct_rank'] = get_pct_ranks(test_df, ['id','cell_type'])","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.057769,"end_time":"2022-08-11T17:36:39.016451","exception":false,"start_time":"2022-08-11T17:36:38.958682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:21.906101Z","iopub.execute_input":"2022-08-11T20:32:21.906443Z","iopub.status.idle":"2022-08-11T20:32:21.962489Z","shell.execute_reply.started":"2022-08-11T20:32:21.906406Z","shell.execute_reply":"2022-08-11T20:32:21.960313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained('../input/ai4code-tokenizers/codebert')\nregressor = get_reg_model('../input/ai4code-models/codebert', pad_token_id=1)\nregressor.load_weights('../input/ai4code-weights/reg-last-0.08864-0.8571.h5')\n\ntest_data = extract_reg_data(test_df, tokenizer, 72, 22, 512)\ntest_dataset = get_dataset(data=get_reg_input(test_data), batch_size=64)\n\nreg_ranks = regressor.predict(test_dataset, verbose=1)[:,0]\ndel regressor, test_data, test_dataset","metadata":{"papermill":{"duration":0.014507,"end_time":"2022-08-11T17:36:39.034086","exception":false,"start_time":"2022-08-11T17:36:39.019579","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:21.964246Z","iopub.execute_input":"2022-08-11T20:32:21.964620Z","iopub.status.idle":"2022-08-11T20:32:50.988469Z","shell.execute_reply.started":"2022-08-11T20:32:21.964587Z","shell.execute_reply":"2022-08-11T20:32:50.987360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = dict(pad_token_id=1, batch_size=64, from_pt=False)\nembedder, model = get_match_model('../input/ai4code-models/unixcoder', **p)\ntokenizer = AutoTokenizer.from_pretrained('../input/ai4code-tokenizers/unixcoder')\nmodel.load_weights('../input/ai4code-weights/match-0.8451.h5')\n\ntest_data = extract_match_data(test_df, tokenizer, 128, 7)\nmarks_dataset = get_dataset(data=test_data['mark_ids'], batch_size=64)\ncodes_dataset = get_dataset(data=test_data['code_ids'], batch_size=64)\nmarks_embs = embedder.predict(marks_dataset, verbose=1)\ncodes_embs = embedder.predict(codes_dataset, verbose=1)\n\npredicted = []\nnb_mark_max_idx = test_data['mark_nb'].max()\nnb_code_max_idx = test_data['code_nb'].max()\nnb_max_idx = max(nb_mark_max_idx, nb_code_max_idx)\n\nfor i in tqdm(range(nb_max_idx + 1)):\n    \n    m = test_data['mark_nb'] == i\n    c = test_data['code_nb'] == i\n    \n    if m.sum() > 0 and c.sum() == 0:\n        predicted.append(test_data['mark_pos'][m])\n    \n    if m.sum() > 0 and c.sum() > 0:\n        scores = pairwise_cosine_similarity(marks_embs[m], codes_embs[c])\n        predicted.append(test_data['code_pos'][c][tf.argmax(scores, axis=1).numpy()])\n\ndel model, test_data, marks_dataset, codes_dataset, marks_embs, codes_embs\nmatch_ranks = np.concatenate(predicted) - 0.001","metadata":{"papermill":{"duration":33.116732,"end_time":"2022-08-11T17:37:12.153469","exception":false,"start_time":"2022-08-11T17:36:39.036737","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:32:50.990301Z","iopub.execute_input":"2022-08-11T20:32:50.991073Z","iopub.status.idle":"2022-08-11T20:33:10.739389Z","shell.execute_reply.started":"2022-08-11T20:32:50.991032Z","shell.execute_reply":"2022-08-11T20:33:10.737537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit(test_df, reg_ranks, match_ranks, rerank_match=True, reg_coef=1.2, match_coef=0.8)","metadata":{"papermill":{"duration":0.035809,"end_time":"2022-08-11T17:37:12.192817","exception":false,"start_time":"2022-08-11T17:37:12.157008","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-11T20:33:10.741441Z","iopub.execute_input":"2022-08-11T20:33:10.741812Z","iopub.status.idle":"2022-08-11T20:33:10.777001Z","shell.execute_reply.started":"2022-08-11T20:33:10.741759Z","shell.execute_reply":"2022-08-11T20:33:10.775785Z"},"trusted":true},"execution_count":null,"outputs":[]}]}