{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport json\nimport numpy as np \nimport pandas as pd\nimport re\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfrom multiprocessing import Pool\n\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom tqdm import tqdm_notebook as tqdm\nimport Levenshtein \n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.linear_model import LogisticRegression\n\nfrom scipy import spatial\nimport lightgbm as lgb\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.over_sampling import RandomOverSampler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepairing data\n\nHere we are going to collect data from json files and format it to the tabular data. We will formulate the problem as a binary classification problem and will try to classify if chosen candidate is an answer."},{"metadata":{},"cell_type":"markdown","source":"UPD: I have moved the code for feature extraction in a separate script https://www.kaggle.com/opanichev/tf2-0-qa-feature-extraction and ran it on my own computer (for some reason it hangs in Kaggle's kernels). I have created a dataset with extracted features https://www.kaggle.com/opanichev/tf20-qa-features so it can be accessible from other kernels."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndtypes = {\n    'CorrectString': str,\n    'CorrectString_short': str,\n    'CandidateString': str\n}\ntrain = pd.read_csv('../input/tf20-qa-features/train_data.csv', dtype=dtypes)\n# test = pd.read_csv('../input/tf20-qa-features/test_data.csv', dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"html_tags = ['<P>', '</P>', '<Table>', '</Table>', '<Tr>', '</Tr>', '<Ul>', '<Ol>', '<Dl>', '</Ul>', '</Ol>', \\\n             '</Dl>', '<Li>', '<Dd>', '<Dt>', '</Li>', '</Dd>', '</Dt>']\nr_buf = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can', 'the', 'a', 'an', 'of', 'in', 'and', 'on', \\\n         'what', 'where', 'when', 'which']\n\n\ndef clean(x, stop_words=[]):\n    for r in html_tags:\n        x = x.replace(r, '')\n    for r in stop_words:\n        x = x.replace(r, '')\n    x = x.lower()\n    x = re.sub(' +', ' ', x)\n    return x\n\n\nfeature_names = [\n    'qa_cos_d', 'qd_cos_d', 'ad_cos_d', \n    'qa_euc_d', 'qd_euc_d', 'ad_euc_d',\n    'qa_lev_d', 'qa_lev_r', 'qa_jar_s', 'qa_jaw_s',\n    'qa_tfidf_score', 'qd_tfidf_score', 'ad_tfidf_score', \n    'document_tfidf_sum', 'question_tfidf_sum', 'answer_tfidf_sum'\n]\n\ndef extract_features(document_tfidf, question_tfidf, answer_tfidf, document, question, answer):\n    qa_cos_d = spatial.distance.cosine(question_tfidf, answer_tfidf)\n    qd_cos_d = spatial.distance.cosine(question_tfidf, document_tfidf)\n    ad_cos_d = spatial.distance.cosine(answer_tfidf, document_tfidf)\n\n    qa_euc_d = np.linalg.norm(question_tfidf - answer_tfidf)\n    qd_euc_d = np.linalg.norm(question_tfidf - document_tfidf)\n    ad_euc_d = np.linalg.norm(answer_tfidf - document_tfidf)\n    \n    qa_lev_d = Levenshtein.distance(question, answer)\n    qa_lev_r = Levenshtein.ratio(question, answer)\n    qa_jar_s = Levenshtein.jaro(question, answer) \n    qa_jaw_s = Levenshtein.jaro_winkler(question, answer)\n    \n    qa_tfidf_score = np.sum(question_tfidf*answer_tfidf.T)\n    qd_tfidf_score = np.sum(question_tfidf*document_tfidf.T)\n    ad_tfidf_score = np.sum(answer_tfidf*document_tfidf.T)\n    \n    document_tfidf_sum = np.sum(document_tfidf)\n    question_tfidf_sum = np.sum(question_tfidf)\n    answer_tfidf_sum = np.sum(answer_tfidf)\n    \n    f = [\n        qa_cos_d, qd_cos_d, ad_cos_d, \n        qa_euc_d, qd_euc_d, ad_euc_d,\n        qa_lev_d, qa_lev_r, qa_jar_s, qa_jaw_s,\n        qa_tfidf_score, qd_tfidf_score, ad_tfidf_score, \n        document_tfidf_sum, question_tfidf_sum, answer_tfidf_sum\n    ]       \n    return f\n\n\ndef process_sample(args):\n    json_data, annotated = args\n    \n    ids = []\n    candidates_str = []\n    targets = []\n    targets_str = []\n    targets_str_short = []\n    features = []\n    rank_features = []\n\n    document = json_data['document_text']\n        \n    # TFIDF for document\n    stop_words = text.ENGLISH_STOP_WORDS.union(['book'])\n    tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words=stop_words)\n    tfidf.fit([document])\n\n    document_tfidf = tfidf.transform([document]).todense()\n\n    # TFIDF for question\n    question = json_data['question_text']\n    question_tfidf = tfidf.transform([question]).todense()\n\n    if annotated:\n        # Collect annotations\n        start_token_true = json_data['annotations'][0]['long_answer']['start_token']\n        end_token_true = json_data['annotations'][0]['long_answer']['end_token']\n\n        # Collect short annotations\n        if json_data['annotations'][0]['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                s_ans = str(json_data['annotations'][0]['short_answers'][0]['start_token']) + ':' + \\\n                    str(json_data['annotations'][0]['short_answers'][0]['end_token'])\n            else:\n                s_ans = ''\n        else:\n            s_ans = json_data['annotations'][0]['yes_no_answer']\n\n    cos_d_buf = []\n    euc_d_buf = []\n    lev_d_buf = []\n\n    doc_tokenized = json_data['document_text'].split(' ')\n    candidates = json_data['long_answer_candidates']\n    candidates = [c for c in candidates if c['top_level'] == True]\n\n    if not annotated or start_token_true != -1:\n        for c in candidates:\n            ids.append(str(json_data['example_id']))\n\n            # TFIDF for candidate answer\n            start_token = c['start_token']\n            end_token = c['end_token']\n            answer = ' '.join(doc_tokenized[start_token:end_token])\n            answer_tfidf = tfidf.transform([answer]).todense()\n\n            # Extract some features\n            f = extract_features(document_tfidf, question_tfidf, answer_tfidf, \n                                 clean(document), clean(question, stop_words=r_buf), clean(answer))\n\n            cos_d_buf.append(f[0])\n            euc_d_buf.append(f[3])\n            lev_d_buf.append(f[6])\n\n            features.append(f)\n\n            if annotated:\n                targets_str.append(str(start_token_true) + ':' + str(end_token_true))\n                targets_str_short.append(s_ans)\n                # Get target\n                if start_token == start_token_true and end_token == end_token_true:\n                    target = 1\n                else:\n                    target = 0\n                targets.append(target)\n                \n            candidates_str.append(str(start_token) + ':' + str(end_token))\n            \n        features = np.array(features)\n        \n        rank_cos_d = np.argsort(np.argsort(cos_d_buf))\n        rank_euc_d = np.argsort(np.argsort(euc_d_buf))\n        rank_lev_d = np.argsort(np.argsort(lev_d_buf))\n        rank_cos_d_ismin = (cos_d_buf == np.nanmin(cos_d_buf)).astype(int)\n        rank_euc_d_ismin = (euc_d_buf == np.nanmin(euc_d_buf)).astype(int)\n        rank_lev_d_ismin = (lev_d_buf == np.nanmin(lev_d_buf)).astype(int)\n        rank_features = np.array([rank_cos_d, rank_euc_d, rank_lev_d, \\\n                                       rank_cos_d_ismin, rank_euc_d_ismin, rank_lev_d_ismin]).T\n\n    return {\n        'ids': ids,\n        'candidates_str': candidates_str,\n        'targets': targets,\n        'targets_str': targets_str,\n        'targets_str_short': targets_str_short,\n        'features': features,\n        'rank_features': rank_features\n    }\n\ndef get_test():\n    ids = []\n    question_tfidfs = []\n    answer_tfidfs = []\n    candidates_str = []\n    features = []\n    rank_features = []\n\n    with open('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', 'r') as json_file:   \n        batch = []\n        batch_cnt = 0\n        for line in tqdm(json_file):\n            json_data = json.loads(line) \n            \n            r = process_sample((json_data, False))\n            if len(r['ids']) > 0:\n                ids += r['ids']\n                candidates_str += r['candidates_str']\n                features.append(r['features'])\n                rank_features.append(r['rank_features'])\n                \n\n    test = pd.DataFrame()\n    test['example_id'] = ids\n    test['CandidateString'] = candidates_str\n\n    features = np.concatenate(features, axis=0)\n    features_df = pd.DataFrame(features)\n    features_df.columns = feature_names\n    test = pd.concat([test, features_df], axis=1)\n\n    rank_features = np.concatenate(rank_features, axis=0)\n    rank_features_df = pd.DataFrame(rank_features)\n    rank_features_df.columns = [f'rank_feature_{i}' for i in range(rank_features.shape[1])]\n    test = pd.concat([test, rank_features_df], axis=1)\n\n    del features, features_df, rank_features, rank_features_df\n    gc.collect()\n    \n    return test\n\n\ntest = get_test()\ntest.to_csv('test_data.csv', index=False)\nprint(f'test.shape: {test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"p_buf = []\nn_splits = 4\n\nkf = GroupKFold(\n    n_splits=n_splits)\n\nerr_buf = []   \n\ncols_to_drop = ['example_id', 'target', 'CorrectString', 'CorrectString_short', 'CandidateString']\n\nX = train.drop(cols_to_drop, axis=1, errors='ignore')\ny = train['target'].values\ng = train['example_id'].values\n\nX_test = test.drop(cols_to_drop, axis=1, errors='ignore')\nid_test = test['example_id'].values\n\nprint(f'X.shape: {X.shape}, y.shape: {y.shape}')\nprint(f'X_test.shape: {X_test.shape}')\n\nn_features = X.shape[1]\n\nlgb_params = {\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'binary_logloss',\n    'max_depth': 16,\n    'learning_rate': 0.005, \n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.9,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'num_threads': 4,\n}\n\nfor fold_i, (train_index, valid_index) in enumerate(kf.split(X, y, g)):\n    print('Fold {}/{}'.format(fold_i + 1, n_splits))\n    params = lgb_params.copy() \n    \n    X_train, y_train = X.iloc[train_index], y[train_index]\n    X_valid, y_valid = X.iloc[valid_index], y[valid_index]\n\n    print(f'X_train.shape: {X_train.shape}, X_valid.shape: {X_valid.shape}')\n    feature_names = list(X_train.columns)\n\n    lgb_train = lgb.Dataset(\n        X_train, \n        y_train, \n        feature_name=feature_names,\n        )\n    lgb_train.raw_data = None\n\n    lgb_valid = lgb.Dataset(\n        X_valid, \n        y_valid,\n        feature_name=feature_names,\n        )\n    lgb_valid.raw_data = None\n\n    model = lgb.train(\n        params,\n        lgb_train,\n        num_boost_round=4000,\n        valid_sets=[lgb_train, lgb_valid],\n        early_stopping_rounds=100, \n        verbose_eval=400, \n    )\n\n    # Feature importance\n    if fold_i == 0:\n        importance = model.feature_importance()\n        model_fnames = model.feature_name()\n        tuples = sorted(zip(model_fnames, importance), key=lambda x: x[1])[::-1]\n        tuples = [x for x in tuples if x[1] > 0]\n        print('Important features:')\n        for i in range(20):\n            if i < len(tuples):\n                print(i, tuples[i])\n            else:\n                break\n\n    # Evaluate model\n    p = model.predict(X.loc[valid_index], num_iteration=model.best_iteration)\n    valid_df = train.loc[valid_index]\n    valid_df['pred'] = p\n    pred_df = valid_df.sort_values('pred', ascending=True).groupby('example_id').tail(1)\n\n    pred_df_long = pred_df[['example_id', 'CorrectString', 'CandidateString']]\n    pred_df_long.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\n    pred_df_long['example_id'] = pred_df_long['example_id'].apply(lambda x: str(x) + '_long')\n\n    pred_df_short = pred_df[['example_id', 'CorrectString_short', 'CandidateString']]\n    pred_df_short.rename({'CorrectString_short': 'CorrectString', 'CandidateString': 'PredictionString'}, \\\n                         axis=1, inplace=True)\n    pred_df_short['example_id'] = pred_df_short['example_id'].apply(lambda x: str(x) + '_short')\n    pred_df_short['PredictionString'] = ''\n\n    pred_df = pd.concat([pred_df_long, pred_df_short], axis=0).sort_values('example_id')\n#     print(pred_df.head(20))\n\n    err = f1_score(pred_df['CorrectString'].fillna('').values, \n                   pred_df['PredictionString'].fillna('').values, \n                   average='micro')\n    print('{} F1: {}'.format(fold_i, err))\n    \n    # Inference on test data\n    p_test = model.predict(X_test[feature_names], num_iteration=model.best_iteration)\n    p_buf.append(p_test)\n    err_buf.append(err)\n\n#     if fold_i >= 0: # Comment this to run several folds\n#         break\n\n    del model, lgb_train, lgb_valid, p\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err_mean = np.mean(err_buf)\nerr_std = np.std(err_buf)\nprint('F1 = {:.4f} +/- {:.4f}'.format(err_mean, err_std))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = train.loc[valid_index]\ntest['pred'] = np.mean(p_buf, axis=0)\npred_df = test.sort_values('pred', ascending=True).groupby('example_id').tail(1)\n\npred_df_long = pred_df[['example_id', 'CandidateString']]\npred_df_long.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\npred_df_long['example_id'] = pred_df_long['example_id'].apply(lambda x: str(x) + '_long')\n\npred_df_short = pred_df[['example_id', 'CandidateString']]\npred_df_short.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\npred_df_short['example_id'] = pred_df_short['example_id'].apply(lambda x: str(x) + '_short')\npred_df_short['PredictionString'] = ''\n# pred_df_short['PredictionString'] = np.nan # https://www.kaggle.com/c/tensorflow2-question-answering/discussion/115836\n\nsubm = pd.concat([pred_df_long, pred_df_short], axis=0).sort_values('example_id')\nsubm.to_csv('submission.csv', index=False)\nprint(f'subm.shape: {subm.shape}')\nsubm.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}