{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport json\nimport numpy as np \nimport pandas as pd\nimport re\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom sklearn.metrics import accuracy_score, f1_score\nfrom tqdm import tqdm_notebook as tqdm\nimport Levenshtein \n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.linear_model import LogisticRegression\n\nfrom scipy import spatial\nimport lightgbm as lgb\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.over_sampling import RandomOverSampler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"html_tags = ['<P>', '</P>', '<Table>', '</Table>', '<Tr>', '</Tr>', '<Ul>', '<Ol>', '<Dl>', '</Ul>', '</Ol>', \\\n             '</Dl>', '<Li>', '<Dd>', '<Dt>', '</Li>', '</Dd>', '</Dt>']\nr_buf = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can', 'the', 'a', 'an', 'of', 'in', 'and', 'on', \\\n         'what', 'where', 'when', 'which'] + html_tags\n\ndef clean(x):\n    x = x.lower()\n    for r in r_buf:\n        x = x.replace(r, '')\n    x = re.sub(' +', ' ', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepairing train dataset\n\nHere we are going to collect data from json files and format it to the tabular data. We will formulate the problem as a binary classification problem and will try to classify if chosen candidate is an answer."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nn_samples = 75000 # Number of samples to read from the train.json\n\n# Read data from train.json and prepare features\nids = []\nquestion_tfidfs = []\nanswer_tfidfs = []\ncandidates_str = []\ntargets = []\ntargets_str = []\ntargets_str_short = []\nfeatures = []\nrank_features = []\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'r') as json_file:\n    cnt = 0\n    for line in tqdm(json_file):\n        json_data = json.loads(line) \n        \n        # TFIDF for document\n        stop_words = text.ENGLISH_STOP_WORDS.union([\"book\"])\n        tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words=stop_words)\n        tfidf.fit([json_data['document_text']])\n\n        # TFIDF for question\n        question = json_data['question_text']\n        question_tfidf = tfidf.transform([question]).todense()\n        \n        # Collect annotations\n        start_token_true = json_data['annotations'][0]['long_answer']['start_token']\n        end_token_true = json_data['annotations'][0]['long_answer']['end_token']\n        \n        # Collect short annotations\n        if json_data['annotations'][0]['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                s_ans = str(json_data['annotations'][0]['short_answers'][0]['start_token']) + ':' + \\\n                    str(json_data['annotations'][0]['short_answers'][0]['end_token'])\n            else:\n                s_ans = ''\n        else:\n            s_ans = json_data['annotations'][0]['yes_no_answer']\n\n        cos_d_buf = []\n        euc_d_buf = []\n        lev_d_buf = []\n        \n        doc_tokenized = json_data['document_text'].split(' ')\n        candidates = json_data['long_answer_candidates']\n        candidates = [c for c in candidates if c['top_level'] == True]\n        \n        if start_token_true != -1:\n            for c in candidates:\n                ids.append(str(json_data['example_id']))\n\n                # TFIDF for candidate answer\n                start_token = c['start_token']\n                end_token = c['end_token']\n                answer = ' '.join(doc_tokenized[start_token:end_token])\n                answer_tfidf = tfidf.transform([answer]).todense()\n\n                # Extract some features\n                cos_d = spatial.distance.cosine(question_tfidf, answer_tfidf)\n                euc_d = np.linalg.norm(question_tfidf - answer_tfidf)\n                lev_d = Levenshtein.distance(clean(question), clean(answer))\n                lev_r = Levenshtein.ratio(clean(question), clean(answer))\n                jar_s = Levenshtein.jaro(clean(question), clean(answer))\n                jaw_s = Levenshtein.jaro_winkler(clean(question), clean(answer))\n                tfidf_score = np.sum(question_tfidf*answer_tfidf.T)\n                question_tfidf_sum = np.sum(question_tfidf)\n                answer_tfidf_sum = np.sum(answer_tfidf)\n\n                features.append([\n                    cos_d, \n                    euc_d, \n                    lev_d, \n                    lev_r, \n                    jar_s, \n                    jaw_s, \n                    tfidf_score, \n                    question_tfidf_sum, \n                    answer_tfidf_sum\n                ])\n                \n                cos_d_buf.append(cos_d)\n                euc_d_buf.append(euc_d)\n                lev_d_buf.append(lev_d)\n\n                targets_str.append(str(start_token_true) + ':' + str(end_token_true))\n                candidates_str.append(str(start_token) + ':' + str(end_token))\n                targets_str_short.append(s_ans)\n\n                # Get target\n                if start_token == start_token_true and end_token == end_token_true:\n                    target = 1\n                else:\n                    target = 0\n                targets.append(target)\n\n            rank_cos_d = np.argsort(cos_d_buf)\n            rank_euc_d = np.argsort(euc_d_buf)\n            rank_lev_d = np.argsort(lev_d_buf)\n            rank_cos_d_ismin = (cos_d_buf == np.nanmin(cos_d_buf)).astype(int)\n            rank_euc_d_ismin = (euc_d_buf == np.nanmin(euc_d_buf)).astype(int)\n            rank_lev_d_ismin = (lev_d_buf == np.nanmin(lev_d_buf)).astype(int)\n            rank_features.append(np.array([rank_cos_d, rank_euc_d, rank_lev_d, \\\n                                           rank_cos_d_ismin, rank_euc_d_ismin, rank_lev_d_ismin]).T)\n\n        cnt += 1\n        if cnt >= n_samples:\n            break\n        \ntrain = pd.DataFrame()\ntrain['example_id'] = ids\ntrain['target'] = targets\ntrain['CorrectString'] = targets_str\ntrain['CorrectString_short'] = targets_str_short\ntrain['CandidateString'] = candidates_str\n\nfeatures = np.array(features)\nfeatures_df = pd.DataFrame(features)\nfeatures_df.columns = [f'feature_{i}' for i in range(features.shape[1])]\ntrain = pd.concat([train, features_df], axis=1)\n\nrank_features = np.concatenate(rank_features, axis=0)\nrank_features_df = pd.DataFrame(rank_features)\nrank_features_df.columns = [f'rank_feature_{i}' for i in range(rank_features.shape[1])]\ntrain = pd.concat([train, rank_features_df], axis=1)\n\ndel features, features_df, \\\n    rank_features, rank_features_df\ngc.collect()\n\ntrain.to_csv('train_data.csv', index=False)\nprint(f'train.shape: {train.shape}')\nprint(f'Mean target: {train.target.mean()}')\ntrain.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preparing test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nids = []\nquestion_tfidfs = []\nanswer_tfidfs = []\ncandidates_str = []\ntargets = []\ntargets_str = []\nfeatures = []\nrank_features = []\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', 'r') as json_file:\n    for line in tqdm(json_file):\n        json_data = json.loads(line) \n        \n        # TFIDF for document\n        stop_words = text.ENGLISH_STOP_WORDS.union([\"book\"])\n        tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words=stop_words)\n        tfidf.fit([json_data['document_text']])\n        \n        # TFIDF for question\n        question = json_data['question_text']\n        question_tfidf = tfidf.transform([question]).todense()\n        \n        doc_tokenized = json_data['document_text'].split(' ')\n        candidates = json_data['long_answer_candidates']\n        candidates = [c for c in candidates if c['top_level'] == True]\n        \n        cos_d_buf = []\n        euc_d_buf = []\n        lev_d_buf = []\n        \n        for c in candidates:\n            ids.append(str(json_data['example_id']))\n            \n            # TFIDF for candidate answer\n            start_token = c['start_token']\n            end_token = c['end_token']\n            answer = ' '.join(doc_tokenized[start_token:end_token])\n            answer_tfidf = tfidf.transform([answer]).todense()\n            \n            # Extract some features\n            cos_d = spatial.distance.cosine(question_tfidf, answer_tfidf)\n            euc_d = np.linalg.norm(question_tfidf - answer_tfidf)\n            lev_d = Levenshtein.distance(clean(question), clean(answer))\n            lev_r = Levenshtein.ratio(clean(question), clean(answer))\n            jar_s = Levenshtein.jaro(clean(question), clean(answer))\n            jaw_s = Levenshtein.jaro_winkler(clean(question), clean(answer))\n            tfidf_score = np.sum(question_tfidf*answer_tfidf.T)\n            question_tfidf_sum = np.sum(question_tfidf)\n            answer_tfidf_sum = np.sum(answer_tfidf)\n\n            features.append([\n                cos_d, \n                euc_d, \n                lev_d, \n                lev_r, \n                jar_s, \n                jaw_s, \n                tfidf_score, \n                question_tfidf_sum, \n                answer_tfidf_sum\n            ])\n\n            cos_d_buf.append(cos_d)\n            euc_d_buf.append(euc_d)\n            lev_d_buf.append(lev_d)\n            \n            candidates_str.append(str(start_token) + ':' + str(end_token))\n        \n        rank_cos_d = np.argsort(cos_d_buf)\n        rank_euc_d = np.argsort(euc_d_buf)\n        rank_lev_d = np.argsort(lev_d_buf)\n        rank_cos_d_ismin = (cos_d_buf == np.nanmin(cos_d_buf)).astype(int)\n        rank_euc_d_ismin = (euc_d_buf == np.nanmin(euc_d_buf)).astype(int)\n        rank_lev_d_ismin = (lev_d_buf == np.nanmin(lev_d_buf)).astype(int)\n        rank_features.append(np.array([rank_cos_d, rank_euc_d, rank_lev_d, \\\n                                       rank_cos_d_ismin, rank_euc_d_ismin, rank_lev_d_ismin]).T)\n        \ntest = pd.DataFrame()\ntest['example_id'] = ids\ntest['CandidateString'] = candidates_str\n\nfeatures = np.array(features)\nfeatures_df = pd.DataFrame(features)\nfeatures_df.columns = [f'feature_{i}' for i in range(features.shape[1])]\ntest = pd.concat([test, features_df], axis=1)\n\nrank_features = np.concatenate(rank_features, axis=0)\nrank_features_df = pd.DataFrame(rank_features)\nrank_features_df.columns = [f'rank_feature_{i}' for i in range(rank_features.shape[1])]\ntest = pd.concat([test, rank_features_df], axis=1)\n\ndel features, features_df, rank_features, rank_features_df\ngc.collect()\n\ntest.to_csv('test_data.csv', index=False)\nprint(f'test.shape: {test.shape}')\ntest.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"p_buf = []\nn_splits = 4\n\nkf = GroupKFold(\n    n_splits=n_splits)\n\nerr_buf = []   \n\ncols_to_drop = ['example_id', 'target', 'CorrectString', 'CorrectString_short', 'CandidateString']\n\nX = train.drop(cols_to_drop, axis=1, errors='ignore')\ny = train['target'].values\ng = train['example_id'].values\n\nX_test = test.drop(cols_to_drop, axis=1, errors='ignore')\nid_test = test['example_id'].values\n\nprint(f'X.shape: {X.shape}, y.shape: {y.shape}')\nprint(f'X_test.shape: {X_test.shape}')\n\nn_features = X.shape[1]\n\nlgb_params = {\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'binary_logloss',\n    'max_depth': 16,\n    'learning_rate': 0.0055, \n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.9,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'num_threads': 5,\n}\n\nfor fold_i, (train_index, valid_index) in enumerate(kf.split(X, y, g)):\n    print('Fold {}/{}'.format(fold_i + 1, n_splits))\n    params = lgb_params.copy() \n    \n    X_train, y_train = X.iloc[train_index], y[train_index]\n    X_valid, y_valid = X.iloc[valid_index], y[valid_index]\n\n    print(f'X_train.shape: {X_train.shape}, X_valid.shape: {X_valid.shape}')\n    feature_names = list(X_train.columns)\n\n    lgb_train = lgb.Dataset(\n        X_train, \n        y_train, \n        feature_name=feature_names,\n        )\n    lgb_train.raw_data = None\n\n    lgb_valid = lgb.Dataset(\n        X_valid, \n        y_valid,\n        feature_name=feature_names,\n        )\n    lgb_valid.raw_data = None\n\n    model = lgb.train(\n        params,\n        lgb_train,\n        num_boost_round=1000,\n        valid_sets=[lgb_train, lgb_valid],\n        early_stopping_rounds=100, \n        verbose_eval=100, \n    )\n\n    # Feature importance\n    if fold_i == 0:\n        importance = model.feature_importance()\n        model_fnames = model.feature_name()\n        tuples = sorted(zip(model_fnames, importance), key=lambda x: x[1])[::-1]\n        tuples = [x for x in tuples if x[1] > 0]\n        print('Important features:')\n        for i in range(20):\n            if i < len(tuples):\n                print(i, tuples[i])\n            else:\n                break\n\n    # Evaluate model\n    p = model.predict(X.loc[valid_index], num_iteration=model.best_iteration)\n    valid_df = train.loc[valid_index]\n    valid_df['pred'] = p\n    pred_df = valid_df.sort_values('pred', ascending=True).groupby('example_id').tail(1)\n\n    pred_df_long = pred_df[['example_id', 'CorrectString', 'CandidateString']]\n    pred_df_long.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\n    pred_df_long['example_id'] = pred_df_long['example_id'].apply(lambda x: x + '_long')\n\n    pred_df_short = pred_df[['example_id', 'CorrectString_short', 'CandidateString']]\n    pred_df_short.rename({'CorrectString_short': 'CorrectString', 'CandidateString': 'PredictionString'}, \\\n                         axis=1, inplace=True)\n    pred_df_short['example_id'] = pred_df_short['example_id'].apply(lambda x: x + '_short')\n    pred_df_short['PredictionString'] = ''\n\n    pred_df = pd.concat([pred_df_long, pred_df_short], axis=0).sort_values('example_id')\n#     print(pred_df.head(20))\n\n    err = f1_score(pred_df['CorrectString'].values, pred_df['PredictionString'].values, average='micro')\n    print('{} F1: {}'.format(fold_i, err))\n    \n    # Inference on test data\n    p_test = model.predict(X_test[feature_names], num_iteration=model.best_iteration)\n    p_buf.append(p_test)\n    err_buf.append(err)\n\n#     if fold_i >= 0: # Comment this to run several folds\n#         break\n\n    del model, lgb_train, lgb_valid, p\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err_mean = np.mean(err_buf)\nerr_std = np.std(err_buf)\nprint('F1 = {:.4f} +/- {:.4f}'.format(err_mean, err_std))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = train.loc[valid_index]\ntest['pred'] = np.mean(p_buf, axis=0)\npred_df = test.sort_values('pred', ascending=True).groupby('example_id').tail(1)\n\npred_df_long = pred_df[['example_id', 'CandidateString']]\npred_df_long.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\npred_df_long['example_id'] = pred_df_long['example_id'].apply(lambda x: str(x) + '_long')\n\npred_df_short = pred_df[['example_id', 'CandidateString']]\npred_df_short.rename({'CandidateString': 'PredictionString'}, axis=1, inplace=True)\npred_df_short['example_id'] = pred_df_short['example_id'].apply(lambda x: str(x) + '_short')\npred_df_short['PredictionString'] = ''\n\nsubm = pd.concat([pred_df_long, pred_df_short], axis=0).sort_values('example_id')\nsubm.to_csv('submission.csv', index=False)\nprint(f'subm.shape: {subm.shape}')\nprint(subm.head(20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}