{"cells":[{"metadata":{"_uuid":"c77c34b81d5e1d99bc91b1909b5f621062be7932"},"cell_type":"markdown","source":"In this kernel I tried to understand the data, especially which languages and characters are used in the texts."},{"metadata":{"trusted":false,"_uuid":"4c37728647925b65248d03879c9b5f90db43edc3"},"cell_type":"code","source":"from pathlib import Path\nimport warnings\nfrom collections import Counter\nimport regex\nimport tqdm\nfrom tqdm import tqdm, tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b53b30dfff3df44fbeb99b9829a89c5502a368eb"},"cell_type":"code","source":"plt.rcParams['figure.figsize'] = (8, 6)\nwarnings.filterwarnings('ignore')\n\n#tqdm_notebook().pandas()\ntqdm().pandas()\n\nDATA_PATH = Path('../input/')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9cd9e03d02582c52a1c59f5f221546257f289a9"},"cell_type":"markdown","source":"### Data Loading"},{"metadata":{"trusted":false,"_uuid":"03b501845332173d9c28b173abef08bbadc0ae3a"},"cell_type":"code","source":"train_df = pd.read_csv(DATA_PATH/'train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c57a311dfe0149b30cf08bd9b01c930861cd6c3b"},"cell_type":"code","source":"test_df = pd.read_csv(DATA_PATH/'test.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5741ed0d1563d40336d396fdce0acc36a5d58e77"},"cell_type":"markdown","source":"### EDA"},{"metadata":{"_uuid":"8a6df2c1ca81e4b50aafbf65cdc84eb7e8eff0c6"},"cell_type":"markdown","source":"**Summary**"},{"metadata":{"trusted":false,"_uuid":"3033ece96c378dfc234fc4322a9a40db837c9a34"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f8393b3d7fd29f45001eeedcf57b1d6673ad5867"},"cell_type":"code","source":"train_df['target'] = train_df['target'].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"09eb44b4278d09dc34a717d22d07b594ccd59bcb"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9cf6381ca9493fd53ff59dd5831195b08e0b5563"},"cell_type":"code","source":"train_df.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e135ca33ad50549be8f9b589a22cd2e6a185a73d"},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8fce907a85a4dcb224c535fcd9d26418c3353348"},"cell_type":"code","source":"test_df.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d6426c8dda5195ae61d99f106ab1023334f504c6"},"cell_type":"code","source":"# Duplicated question texts\ntrain_df.duplicated('question_text').sum(), test_df.duplicated('question_text').sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4b4fb42c2996fa9f61dabb3347555a0f478b227"},"cell_type":"markdown","source":"**Get copy of train/test datasets**"},{"metadata":{"trusted":false,"_uuid":"93efe5f8bc60b07a99d6ad31774a5183c1be6211"},"cell_type":"code","source":"X_train = train_df.copy()\nX_test = test_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"542dfac37ee7b41563b1851fcada725dd2b2a156"},"cell_type":"markdown","source":"**Target distribution**"},{"metadata":{"trusted":false,"_uuid":"0ad145fb29f00a01a4316323544fb2e076d4f2df"},"cell_type":"code","source":"X_train['target'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07b5352d0cba15867e48f87ca45bb25eed505eb6"},"cell_type":"markdown","source":"**Question ids distribution**"},{"metadata":{"trusted":false,"_uuid":"dbd3ead51ec6ced384487d131515badc9dc68986"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"25f4dd80c594ca48900d98a8b1f3df00eb969306"},"cell_type":"code","source":"X_test['qid'].str.len().unique()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48e7cf63704c06e19188a115b4d148ec066bf335"},"cell_type":"markdown","source":"Rolling mean proportion of bad questions by first characters of qid"},{"metadata":{"trusted":false,"_uuid":"ab351dfa1fb7181e295df5ee87d06cf0e6112380"},"cell_type":"code","source":"X_train.groupby(X_train['qid'].str[:1])['target'].apply(lambda x: np.sum(x.astype('int8')) / len(x)).rolling(1).mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d40baca272cf7f1b47c1ced4752ae1052b9eb09a"},"cell_type":"code","source":"X_train.groupby(X_train['qid'].str[:2])['target'].apply(lambda x: np.sum(x.astype('int8')) / len(x)).rolling(10).mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d23f23d1f5da575df4fafb4a4d9ce7dd8d58a0b1"},"cell_type":"code","source":"X_train.groupby(X_train['qid'].str[:3])['target'].apply(lambda x: np.sum(x.astype('int8')) / len(x)).rolling(100).mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"71e805c2b9c8e890f8fce5bfd3f45244c4a687e5"},"cell_type":"code","source":"X_train.groupby(X_train['qid'].str[:4])['target'].apply(lambda x: np.sum(x.astype('int8')) / len(x)).rolling(1000).mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2c4cb04c244a53bf328b8742a8fdd5bdd8a6eaa"},"cell_type":"markdown","source":"**Length of questions**"},{"metadata":{"trusted":false,"_uuid":"5b9a4146758b2a0ce1a0ebdf6724a6a0ea110d20"},"cell_type":"code","source":"X_train['question_length'] = X_train['question_text'].str.len()\nX_test['question_length'] = X_test['question_text'].str.len()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3ed3cd5ce4411acb011ecc4dc6d76a063f21915"},"cell_type":"markdown","source":"Question length in train/test datasets"},{"metadata":{"trusted":false,"_uuid":"f062f3409b70b42dcbda5e702c35b0d894b8d942"},"cell_type":"code","source":"fig, axes = plt.subplots(2, 1, figsize=(16, 6), sharex=True)\nsns.boxplot('question_length', data=X_train, ax=axes[0])\nsns.boxplot('question_length', data=X_test, ax=axes[1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b52b97241e41746ecb88757c927e183550259c38"},"cell_type":"markdown","source":"Density of question length in train/test datasets"},{"metadata":{"trusted":false,"_uuid":"5a543aa536390cb91ef35e530495fd9e06e14e98"},"cell_type":"code","source":"fig, axes = plt.subplots(figsize=(16, 6), sharex=True)\nsns.kdeplot(X_train['question_length'], label='Train')\nsns.kdeplot(X_test['question_length'], label='Test')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce2f9289033623f75d7daea5eebad2fa80d288d9"},"cell_type":"markdown","source":"Aggregate statistics for question length in train/test datasets"},{"metadata":{"trusted":false,"_uuid":"e4e0fc316a8b3a4a294fa103a5de2a89f3d9c54d"},"cell_type":"code","source":"agg_stats = ['min', 'max', 'mean', 'std', 'median']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"32930c1e0eddde760ded3ee1b78c5db5bf2c3f5b"},"cell_type":"code","source":"X_train['question_length'].agg(agg_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6f3a588b9bf63bea71e5e3da11af760826406cfc"},"cell_type":"code","source":"X_test['question_length'].agg(agg_stats)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c568a72dcffb2738a9993feec2bd69c51325e250"},"cell_type":"markdown","source":"Question length distribution by target"},{"metadata":{"trusted":false,"_uuid":"68754358c4dcb5877da852328b8b4cab2219bd95"},"cell_type":"code","source":"sns.boxplot('question_length', 'target', data=X_train, orient='h')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da413e13bc045307004198fa7745858c1da75443"},"cell_type":"markdown","source":"There are few questions in "},{"metadata":{"trusted":false,"_uuid":"808c3f5fb95c0482c7fb16bc6e97332365cd6d2c"},"cell_type":"code","source":"X_train.loc[X_train['question_length'] < 11, 'target'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"34a08fcee5a851742a89d494bcfdf83f751c71b5"},"cell_type":"code","source":"X_train.loc[X_train['question_length'] > 588, 'target'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a587d72aaf45cd9ca61d2b082bea54f023421efe"},"cell_type":"code","source":"questions_outliers = X_train.loc[(X_train['question_length'] < 11) | (X_train['question_length'] > 588)]\nquestions_outliers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73b8378eccc438ab845d1ae7a0f2f699da3e252c"},"cell_type":"markdown","source":"So the proportion of bad questions for questions with very small or large lengths is much larger than proportion for the whole dataset"},{"metadata":{"_uuid":"828eea474aaec39545e2192c23c61d5c1fa64e6a"},"cell_type":"markdown","source":"Let's try to compare distribution of question lengths in both datasets and also for bad and qood questions"},{"metadata":{"trusted":false,"_uuid":"6b3d40e256d00fb502649d927e1becd41da3edd0"},"cell_type":"code","source":"# Original questions lengths distribution\nplt.figure(figsize=(14, 6))\nsns.kdeplot(X_train.loc[X_train['target'] == 0, 'question_length'], label='Train - good')\nsns.kdeplot(X_train.loc[X_train['target'] == 1, 'question_length'], label='Train - bad')\nsns.kdeplot(X_train['question_length'], label='Train')\nsns.kdeplot(X_test['question_length'], label='Test')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6bc70e515ac6b7e77293345599caf537b5376cfe"},"cell_type":"code","source":"X_train.groupby('target')['question_length'].agg(agg_stats)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b69dafc2110ffe8936192aff76420da17aacb21"},"cell_type":"markdown","source":"The density is different only for question length of bad questions"},{"metadata":{"_uuid":"51198255a149ca2fb0a0e96e713546cca70e717d"},"cell_type":"markdown","source":"Let's see the proportion of bad questions by length"},{"metadata":{"trusted":false,"_uuid":"a858ac2986ad9aab6a2ddffc9bc69a183c2d7f45"},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nX_train.groupby('question_length')['target'].apply(lambda x: np.sum(x.astype('int')) / len(x)).plot()\nsns.kdeplot(X_train['question_length'], label='Train')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f9a1f6baa2a1f6b7ddca7f7294660f8400262d15"},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nX_train.groupby('question_length')['target'].apply(lambda x: np.sum(x.astype('int')) / len(x)).rolling(10).mean().plot()\nsns.kdeplot(X_train['question_length'], label='Train')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"eaf918cfa4064f24cbe94a4174a6594b8bfdbaaa"},"cell_type":"code","source":"train_question_lengths_bins = pd.cut(X_train['question_length'], bins=5, include_lowest=True)\nX_train.groupby(train_question_lengths_bins)['target'].apply(lambda x: np.sum(x.astype('int8')) / len(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"adbb39f24c97c058809f85c6f03f671a86b47e06"},"cell_type":"markdown","source":"Almost all questions in train dataset with length less than 10 or greater than 800 are bad"},{"metadata":{"trusted":false,"_uuid":"6d65974371fc3af68b6836fc6ccefe70d3075457"},"cell_type":"code","source":"X_train[(X_train['question_length'] < 10) | (X_train['question_length'] > 800)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67ec1e7a2662431fa887dd6de88b48f5a657e753"},"cell_type":"markdown","source":"There are no questions in test dataset less than 11 and greater than 588"},{"metadata":{"trusted":false,"_uuid":"85aa1b2f9ef79397b4b9b8d21ddbebaa4655962b"},"cell_type":"code","source":"X_test.loc[(X_test['question_length'] < 11) | (X_test['question_length'] > 588)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da58c698fe55f7024fc910ed2c3b1cdace4f4f49"},"cell_type":"markdown","source":"After removing questions less than 11 characters and greater than 588 characters, distributions are almost equal"},{"metadata":{"trusted":false,"_uuid":"44d3c46caedc728503befee6d5ff3b55ee8f9c0d"},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsns.kdeplot(X_train.loc[X_train['target'] == 0, 'question_length'], label='Train - good')\n# sns.kdeplot(X_train.loc[X_train['target'] == 1, 'question_length'], label='Train - bad')\nsns.kdeplot(X_train.loc[(X_train['question_length'] <= 588) & (X_train['question_length'] >= 11),\n                        'question_length'], label='Train - bad (no outliers)')\nsns.kdeplot(X_train['question_length'], label='Train')\nsns.kdeplot(X_test['question_length'], label='Test')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0ab457f994eb529593558dead731204fa46439b"},"cell_type":"markdown","source":"**First word distribution**"},{"metadata":{"_uuid":"0d45b10c44d715c8ad692eb292e96ac7d6761654"},"cell_type":"markdown","source":"Let's see if there any dependency between first word of the question and target"},{"metadata":{"trusted":false,"_uuid":"17a99c2d464982844072df7137bfab9a25f3000f"},"cell_type":"code","source":"X_train['first_word'] = X_train['question_text'].str.extract('(.*?)\\s.*') \nX_test['first_word'] = X_test['question_text'].str.extract('(.*?)\\s.*')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"00953866f0cb43e033a7fcf8b07b45bbd10fdde6"},"cell_type":"code","source":"X_train['first_word'].value_counts(normalize=True).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"005586084936e708925efffa067d77c3d8117e4a"},"cell_type":"code","source":"X_train[X_train['first_word'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7fc89adad18456aef7e64bc015263cd80d0d3266"},"cell_type":"code","source":"X_test['first_word'].value_counts(normalize=True).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5ae7b90ab352dce77de653e5d6548bf28e303d3d"},"cell_type":"code","source":"X_test[X_test['first_word'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"57dad3dc897f3055e938fb22aa592236829e267b"},"cell_type":"code","source":"first_words_0 = X_train.loc[X_train['target'] == 0, 'first_word'].value_counts(normalize=True)\nfirst_words_1 = X_train.loc[X_train['target'] == 1, 'first_word'].value_counts(normalize=True)\nfirst_words_df = pd.concat([first_words_0, first_words_1], axis=1, join='outer')\n\nfirst_words_df.columns = ['good', 'bad']\nfirst_words_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"867cc91c786c8d0e3f1c3cafa486bddb050ade17"},"cell_type":"markdown","source":"Top good first words"},{"metadata":{"trusted":false,"_uuid":"d1a2bd51374f76c80edd9a71832310a3b23a6910"},"cell_type":"code","source":"first_words_df.sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7276c551ca1c608da15f2ee51abb15d2a4ecc6a1"},"cell_type":"markdown","source":"Top bad first words"},{"metadata":{"trusted":false,"_uuid":"86be60d6a9e9d4361d67633b34e0008e196c1427"},"cell_type":"code","source":"first_words_df.sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07e24ba875f526c50b480c9da226dfc56dab47da"},"cell_type":"markdown","source":"Let's check which first words have the largest difference in proportions of bad/good questions"},{"metadata":{"trusted":false,"_uuid":"ea591ddc0cd5f01b9d5f11178e9097bb55840779"},"cell_type":"code","source":"def color_cells(value):\n    if value < 0:\n        color = 'red'\n    elif value > 0:\n        color = 'green'\n    else:\n        color = 'black'\n\n    return f'color: {color}'","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"269669775b556c15cdd38696eec5db3ef4532605"},"cell_type":"code","source":"# Top first words with maximum absolute proportion difference by target\nfirst_words_df['diff'] = first_words_df['bad'] - first_words_df['good']\nfirst_words_df['abs_diff'] = np.abs(first_words_df['bad'] - first_words_df['good'])\n\ntop_first_words_df = first_words_df[first_words_df['abs_diff'] >= 0.01].sort_values('diff', ascending=False)\ntop_first_words_df.style.applymap(color_cells, subset=['diff'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d1caaa0e61bf5fc02eff9b653c89fcfb9b3e7b1a"},"cell_type":"code","source":"top_first_words_df['diff'][::-1].plot.barh();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37d4f58557b3d0331832cfe1559037a5a6a1ba6f"},"cell_type":"markdown","source":"Top good first words (not in bad questions)"},{"metadata":{"trusted":false,"_uuid":"ee528f5a215d8e9f8162c634d433d76da8a83a78"},"cell_type":"code","source":"first_words_df[first_words_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8aca8f257037eb27c5e9c4a6289aa205880d225e"},"cell_type":"code","source":"good_only_first_words = first_words_df[first_words_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].index\ngood_only_first_words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67cd230928f649ce79928e6268c8e39a226f94b5"},"cell_type":"markdown","source":"Top bad first words (not in good questions)"},{"metadata":{"trusted":false,"_uuid":"1340010eef6bb65a9c3371f55a95b2b193d5222b"},"cell_type":"code","source":"first_words_df[first_words_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c84ddc516fbd783f71a00fffdd24b52c9f774687"},"cell_type":"code","source":"bad_only_first_words = first_words_df[first_words_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].index\nbad_only_first_words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed96627216cef58c373699870ed54f41aa224393"},"cell_type":"markdown","source":"Examples with good/bad first words in the test data"},{"metadata":{"trusted":false,"_uuid":"c30aff99a7021ae1afa01c24dcb90d719fc3f8ca"},"cell_type":"code","source":"X_test[X_test['first_word'].isin(good_only_first_words)].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c9291eff6b53616beaed779660253e9362158372"},"cell_type":"code","source":"X_test[X_test['first_word'].isin(bad_only_first_words)].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bff0f7c58c4151b8db2b4de40039edd1087c61d1"},"cell_type":"markdown","source":"**Conclusion**\n- It seems that \"Why\" is often used in bad question as first word, and the words \"What\", \"How\" - in good questions\n- Only bad questions contain many words about politics, terrorism etc."},{"metadata":{"_uuid":"bbd91041f51d0f929f23633e6ade020a1ebb51d6"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"16cf73ad64caff6f43acd51f2bffd1fc9d757367"},"cell_type":"markdown","source":"**Last character distribution**"},{"metadata":{"_uuid":"1f29e046033d474b618fe60b6f15aaa772b3504f"},"cell_type":"markdown","source":"Let's check hypothesis if last character has relationship with the target"},{"metadata":{"trusted":false,"_uuid":"5cf02b7bc85affeb3fb3ee44236b9633fe3025f2"},"cell_type":"code","source":"X_train['last_char'] = X_train['question_text'].str[-1]\nX_test['last_char'] = X_test['question_text'].str[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7074434b629b772597bf9371ef69e93e3d007588"},"cell_type":"code","source":"X_train['last_char'].value_counts(normalize=True).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"015488f370bd384ba2996c33dafb4c35eafa05eb"},"cell_type":"code","source":"X_test['last_char'].value_counts(normalize=True).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"539d424a93367fcbfc99e5519f81d96ecfaa9132"},"cell_type":"code","source":"last_chars_0 = X_train.loc[X_train['target'] == 0, 'last_char'].value_counts(normalize=True)\nlast_chars_1 = X_train.loc[X_train['target'] == 1, 'last_char'].value_counts(normalize=True)\nlast_chars_df = pd.concat([last_chars_0, last_chars_1], axis=1, join='outer')\n\nlast_chars_df.columns = ['good', 'bad']\nlast_chars_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11282e84335ef86035337c3cb4dc38e0fbf90b89"},"cell_type":"markdown","source":"Top good last characters"},{"metadata":{"trusted":false,"_uuid":"033e27d3f63180ec60ffb124ff8529362fac9c2b"},"cell_type":"code","source":"last_chars_df.sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"574c9af8b4995e958461a6ece652567368bc1c10"},"cell_type":"markdown","source":"Top bad last characters"},{"metadata":{"trusted":false,"_uuid":"4948ff5bd8928b04528b455b8aa9f80e74c5a547"},"cell_type":"code","source":"last_chars_df.sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc555c947e6b3fd329f075cb0ea1eb21c8e77a5e"},"cell_type":"markdown","source":"Top last characters with maximum absolute proportion difference by target"},{"metadata":{"trusted":false,"_uuid":"9df95ba5d0bb6ba9feaa8b95d7abaa1b800a43ca"},"cell_type":"code","source":"last_chars_df['diff'] = last_chars_df['bad'] - last_chars_df['good']\nlast_chars_df['abs_diff'] = np.abs(last_chars_df['bad'] - last_chars_df['good'])\n\ntop_last_chars_df = last_chars_df[last_chars_df['abs_diff'] > 0.001].sort_values('diff', ascending=False)\ntop_last_chars_df.style.applymap(color_cells, subset=['diff'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"537223a5691219b6ebd49f5408003c4bbf85a068"},"cell_type":"code","source":"top_last_chars_df['diff'][::-1].plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5a80c4b1a3a29e5aa92ac2904390ebcf3113c28"},"cell_type":"markdown","source":"Top good last characters (not in bad questions)"},{"metadata":{"trusted":false,"_uuid":"8f0d837f7013ea2e9da4ab2272990f57a239e0dc"},"cell_type":"code","source":"last_chars_df[last_chars_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1232478508fb9f891ad66586565bb61eaf68b86e"},"cell_type":"code","source":"good_only_last_chars = last_chars_df[last_chars_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].index.values\ngood_only_last_chars","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6353c0dff67e61a7d634fb40818863a2f80fd9f"},"cell_type":"markdown","source":"Top bad last characters (not in good questions)"},{"metadata":{"trusted":false,"_uuid":"2137b7b54d7948805644e28c73b8fc5fadfbe572"},"cell_type":"code","source":"last_chars_df[last_chars_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d347be3edd2dc2596609dcdaa335550cd99dcb35"},"cell_type":"code","source":"bad_only_last_chars = last_chars_df[last_chars_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].index.values\nbad_only_last_chars","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08dbd8e1347687f3aa79243c9992a03313775054"},"cell_type":"markdown","source":"**Conclusion**\n- As expected, question mark is used more often for good questions, but for bad questions characters like dot, exclamation mark and space are used more often"},{"metadata":{"_uuid":"669f935d544d65c75ceac900fb29750b53e56ffe"},"cell_type":"markdown","source":"**Character set distribution**"},{"metadata":{"_uuid":"5f1c68b8a04b5898820fd8e59d1552982c2b7684"},"cell_type":"markdown","source":"The main part of this EDA - trying to understand how multilingual the dataset is and which types of characters are used.\nFor this goal I've used excellent python package regex (https://pypi.org/project/regex/), which supports much more features than standard package re."},{"metadata":{"_uuid":"36915dce7b17d4fc7fb650b7e5f84211d82275d8"},"cell_type":"markdown","source":"Let's start with overall set of characters in train and test datasets"},{"metadata":{"trusted":false,"_uuid":"dcdefd20a55c47986430c5d8a32c5e57b02e71f8"},"cell_type":"code","source":"train_chars_freq = Counter(X_train['question_text'].str.cat())\ntrain_good_chars_freq = Counter(X_train.loc[X_train['target'] == 0, 'question_text'].str.cat())\ntrain_bad_chars_freq = Counter(X_train.loc[X_train['target'] == 1, 'question_text'].str.cat())\ntest_chars_freq = Counter(X_test['question_text'].str.cat())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4fdb6dde0807394295bf05ec7deca9663e488394"},"cell_type":"code","source":"train_chars_len = len(X_train['question_text'].str.cat())\ntrain_good_chars_len = len(X_train.loc[X_train['target'] == 0, 'question_text'].str.cat())\ntrain_bad_chars_len = len(X_train.loc[X_train['target'] == 1, 'question_text'].str.cat())\ntest_chars_len = len(X_test['question_text'].str.cat())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00359e775a3a104951cdbf484d4f7e83e3cb4e50"},"cell_type":"markdown","source":"Characters vocabulary sizes"},{"metadata":{"trusted":false,"_uuid":"50a7b4a688675e951d6c0f4fa53d0c44e00fd264"},"cell_type":"code","source":"len(set(train_chars_freq)), len(set(train_good_chars_freq)), len(set(train_bad_chars_freq)), len(set(test_chars_freq))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"224fa16f1fbded814185735f798ee6e3432ec629"},"cell_type":"code","source":"train_chars = set(train_chars_freq)\ntrain_good_chars = set(train_good_chars_freq)\ntrain_bad_chars = set(train_bad_chars_freq)\ntest_chars = set(test_chars_freq)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b31c68917af3c414d201d9376511a87e860aab4"},"cell_type":"markdown","source":"Used characters in train dataset which are not in test dataset"},{"metadata":{"trusted":false,"_uuid":"52deee89783eb6ef2c07df866889b70b560fb9c0"},"cell_type":"code","source":"print(sorted(train_chars - test_chars))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b9b3302e41a3ff7c2307ed1f19356a4f16dfd3b"},"cell_type":"markdown","source":"Used characters in test dataset which are not in train dataset"},{"metadata":{"trusted":false,"_uuid":"adebeda2710e89c4f25506b0ba1364043f506fc0"},"cell_type":"code","source":"print(sorted(test_chars - train_chars))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0622107afbd712915d8473f6418830eb0cfedec4"},"cell_type":"markdown","source":"Used characters in good questions which are not in bad questions"},{"metadata":{"trusted":false,"_uuid":"3fe6a3284daf01f18d11fbdae84a0dbb5b4c669f"},"cell_type":"code","source":"print(sorted(train_good_chars - train_bad_chars))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77d5744f355daf0af843168cbc7804991ce7e635"},"cell_type":"markdown","source":"Used characters in bad questions which are not in good questions"},{"metadata":{"trusted":false,"_uuid":"516f1b489a91f3dd9ba49b04607e1365d41d0b05"},"cell_type":"code","source":"print(sorted(set(train_bad_chars) - set(train_good_chars)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"43d60271ba17561fad9ab1214aee8b8ddd7843a9"},"cell_type":"code","source":"train_chars_df = pd.Series(dict(train_chars_freq)) / train_chars_len\ntest_chars_df = pd.Series(dict(test_chars_freq)) / test_chars_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"436b132cbd39a6677fee82f1b7eaed6efd6aa83e"},"cell_type":"code","source":"chars_df = pd.concat([train_chars_df, test_chars_df], axis=1, join='outer')\nchars_df.columns = ['train', 'test']\nchars_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77619bc15949fea80c8d7b913f3fdfe69a502bd0"},"cell_type":"markdown","source":"Top train characters"},{"metadata":{"trusted":false,"_uuid":"56cec1bb846337c64674e89baa8ca6e4744f2bff"},"cell_type":"code","source":"chars_df.sort_values('train', ascending=False)[['train']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5794ddff381e45efd2925573ffff074e20ff25f"},"cell_type":"markdown","source":"Top test characters"},{"metadata":{"trusted":false,"_uuid":"36c668d01cbee6912e4fa3d90146238e48081c72"},"cell_type":"code","source":"chars_df.sort_values('test', ascending=False)[['test']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62f837ab0ab9009c73aa74f02c388f68724daca7"},"cell_type":"markdown","source":"Top characters with maximum absolute proportion difference between train and test"},{"metadata":{"trusted":false,"_uuid":"56a9bbdccae58e3207850cc09956a16b148fc0b7"},"cell_type":"code","source":"chars_df['diff'] = chars_df['train'] - chars_df['test']\nchars_df['abs_diff'] = np.abs(chars_df['train'] - chars_df['test'])\n\ntop_chars_df = chars_df[chars_df['abs_diff'] > 0.0001].sort_values('diff', ascending=False)\ntop_chars_df.style.applymap(color_cells, subset=['diff'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b645e94d38209dd265de76f25e87d7883f47df15"},"cell_type":"code","source":"top_chars_df['diff'][::-1].plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b16aac849486119a92ac6e217c7920c2e8c0d53"},"cell_type":"markdown","source":"Top characters in train dataset (not in test dataset)"},{"metadata":{"trusted":false,"_uuid":"a46ab9c01c6a876b8c21fbb23b4bd7cafbacd015"},"cell_type":"code","source":"chars_df[chars_df['test'].isnull()].sort_values('train', ascending=False)[['train']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e0e6aac4efd308fb16d015b17570c55c8505aba6"},"cell_type":"code","source":"train_only_chars = chars_df[chars_df['test'].isnull()].sort_values('train', ascending=False)[['train']].index.values.tolist()\nprint(train_only_chars)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ce837b5691ae079fa7dc8816b2ffa59eedfc3d3"},"cell_type":"markdown","source":"Top characters in test dataset (not in train dataset)"},{"metadata":{"trusted":false,"_uuid":"11fb812662abf397a3cd2e0616dd6a3f8c2aeddf"},"cell_type":"code","source":"chars_df[chars_df['train'].isnull()].sort_values('test', ascending=False)[['test']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0852ec2b681bcbf8961c81de5180f469481b7ad9"},"cell_type":"code","source":"test_only_chars = chars_df[chars_df['train'].isnull()].sort_values('test', ascending=False)[['test']].index.values\ntest_only_chars","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1f3931759c9d0eb7dc2fc9607f84ff567ef3bb23"},"cell_type":"code","source":"train_good_chars_df = pd.Series(dict(train_good_chars_freq)) / train_good_chars_len\ntrain_bad_chars_df = pd.Series(dict(train_bad_chars_freq)) / train_bad_chars_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fdd75ba1a6b6207cf0f1e4cf095ce381836f4620"},"cell_type":"code","source":"chars_df = pd.concat([train_good_chars_df, train_bad_chars_df], axis=1, join='outer')\nchars_df.columns = ['good', 'bad']\nchars_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f3c5148e96db6e8603ac12bed612cf842591def"},"cell_type":"markdown","source":"Top characters in good questions"},{"metadata":{"trusted":false,"_uuid":"1efd3612f848194d13330d9c5bdb980cad6859b5"},"cell_type":"code","source":"chars_df.sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fddeca2be82bf8474d0aea2a1339f3db3d61d12"},"cell_type":"markdown","source":"Top characters in bad questions"},{"metadata":{"trusted":false,"_uuid":"fce5a9493f8c26c7b6adc3ab6da112a679de8551"},"cell_type":"code","source":"chars_df.sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32e2adce7457f341580537d4dc12330cd1645265"},"cell_type":"markdown","source":"Top characters with maximum absolute proportion difference by target"},{"metadata":{"trusted":false,"_uuid":"1d46f368198c2fdb62338dab4d2d18cb7409061a"},"cell_type":"code","source":"chars_df['diff'] = chars_df['bad'] - chars_df['good']\nchars_df['abs_diff'] = np.abs(chars_df['bad'] - chars_df['good'])\n\ntop_chars_df = chars_df[chars_df['abs_diff'] > 0.001].sort_values('diff', ascending=False)\ntop_chars_df.style.applymap(color_cells, subset=['diff'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc53c1b3d77de6c382330a3b4a249aac05043227"},"cell_type":"code","source":"top_chars_df['diff'][::-1].plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d38b221a3a7c20f2f5897a92ff38b09946e0ccbc"},"cell_type":"markdown","source":"Top characters in good questions (not in bad questions)"},{"metadata":{"trusted":false,"_uuid":"9f6ae5a9b3c07d07d52e9bd42ffa02af24d0e0c2"},"cell_type":"code","source":"chars_df[chars_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b0bfb01b6de701234bb49072b2b2298c7893c5cf"},"cell_type":"code","source":"good_only_chars = chars_df[chars_df['bad'].isnull()].sort_values('good', ascending=False)[['good']].index.values.tolist()\nprint(good_only_chars)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73bab8ada0d154c4d9a96269a06fbd72b1b6a021"},"cell_type":"markdown","source":"Top characters in bad questions (not in good questions)"},{"metadata":{"trusted":false,"_uuid":"35d6f8b09c22ed4b98f320fd64b02aec7900e637"},"cell_type":"code","source":"chars_df[chars_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"de6a01a3467c69febbd222aafd1b29f29f098833"},"cell_type":"code","source":"bad_only_chars = chars_df[chars_df['good'].isnull()].sort_values('bad', ascending=False)[['bad']].index.values\nbad_only_chars","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7b25af5eb44710d3c5069bce6fd9cb45982f7c4"},"cell_type":"markdown","source":"**Character set types**"},{"metadata":{"_uuid":"173ca88e8138e95523f016ded182342d648327f0"},"cell_type":"markdown","source":"Among other features, package regex has the feature to find unicode categories, scripts and blocks (more about this you can read here - https://www.regular-expressions.info/unicode.html)  \nLet's use categories and scripts and find which type of characters and from which language are used in train/test dataset  \nSome of the categories and scripts were not useful (raise an exception or gives the empty result), so I omit those types"},{"metadata":{"_uuid":"75e02dc51e8a5ca95a095228d8e6a327958f3905"},"cell_type":"markdown","source":"Unicode Categories"},{"metadata":{"trusted":false,"_uuid":"86af9dd15c253108f3c3c01a78f514bf2ae8c130"},"cell_type":"code","source":"X_train['question_text'].loc[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6523ff355561f00df0b567ee2fb4a90304509386"},"cell_type":"code","source":"# Any kind of letter from any language\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0324a73dd6b8437791797491dfa1720fbd31416f"},"cell_type":"code","source":"# A lowercase letter that has an uppercase variant\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Lowercase_Letter}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"95c8b6aa5cd16715fc33de2d55cbdf6c510fa3a8"},"cell_type":"code","source":"# An uppercase letter that has a lowercase variant\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Uppercase_Letter}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9e355ae3d6b64ea502769d509cb6cdac370c2c59"},"cell_type":"code","source":"# Alternative method for title-case words\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\b\\p{Uppercase_Letter}\\p{Lowercase_Letter}+\\b', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3ad952025d45e0344bafed80e6f6297c7662b5a0"},"cell_type":"code","source":"# A letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cased_Letter}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2388eb72755e50a206bca711e05076c208b38d11"},"cell_type":"code","source":"# A special character that is used like a letter\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Letter}', x)).loc[[14498, 18460, 81490]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cacf9f61beb46c50f02289a2cd1134c6fa5b05ca"},"cell_type":"code","source":"# A letter or ideograph that does not have lowercase and uppercase variants\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Letter}', x)).loc[[905, 1360, 2194]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8f8240e2b3793ba9efcf0eb9e28d097a5484d69e"},"cell_type":"code","source":"# A character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mark}', x)).loc[[441, 2194, 9479]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5402c347e724d16cdc3c98f02536f6cf329dacf7"},"cell_type":"code","source":"# A character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Non_Spacing_Mark}', x)).loc[[441, 2194, 9479]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"407608432c93aaed9f7a2a24f9bbb6b122996dae"},"cell_type":"code","source":"# Any kind of whitespace or invisible separator\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Separator}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a9a71473148a3ce12a9b015024c7043903a78c83"},"cell_type":"code","source":"# A whitespace character that is invisible, but does take up space\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Space_Separator}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"64402e30da1cec87bdd75a7e3e594dfc35a4d6c9"},"cell_type":"code","source":"# Math symbols, currency signs, dingbats, box-drawing characters, etc.\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Symbol}', x)).loc[[83, 94, 97]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2a94fd3ead0a977829caa81e6580a9102260f0f7"},"cell_type":"code","source":"# Any mathematical symbol\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Math_Symbol}', x)).loc[[94, 97, 274]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a9f3a5bc07c2b3efea08ed2825a2ff13b62d9b87"},"cell_type":"code","source":"# Any currency sign\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Currency_Symbol}', x)).loc[[83, 345, 1897]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"218b920ad7ec5d1fde2fc8d810defa8178a42167"},"cell_type":"code","source":"# A combining character (mark) as a full character on its own\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Symbol}', x)).loc[[540, 1035, 1322]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c81575d99ce805dd3937b86fd5caed0c524d0548"},"cell_type":"code","source":"# Various symbols that are not math symbols, currency signs, or combining characters\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Symbol}', x)).loc[[2805, 2922, 4480]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"abb1af9ecf1409eebf9f0e85ba9b84902ba68693"},"cell_type":"code","source":"# Any kind of numeric character in any script\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Number}', x)).loc[[0, 14, 27]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17fb7501219780c4a665b8fe9d31caf87b460a1f"},"cell_type":"code","source":"# A number that looks like a letter, such as a Roman numeral\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter_Number}', x)).loc[887588]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ce1622a934b3a9190029e17f37ab4b94caa6bfb7"},"cell_type":"code","source":"# A superscript or subscript digit, or a number that is not a digit 0–9 (excluding numbers from ideographic scripts)\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Number}+', x)).loc[[3605, 12830, 29910]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d278e0c38278f0ca596bca4ec5d71784ebdcbcc9"},"cell_type":"code","source":"# Any kind of punctuation character\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Punctuation}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7582f4108e61b0797249e58a8e5ed1173d61467f"},"cell_type":"code","source":"# Any kind of hyphen or dash\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Dash_Punctuation}', x)).loc[[33, 48, 94]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0306ee519e87390827bf3b13ac1d8ae987ff6b3b"},"cell_type":"code","source":"# Any kind of opening bracket\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Open_Punctuation}', x)).loc[[35, 53, 86]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"eb1338d501a7e8036900bde21460ddf0cb3efd58"},"cell_type":"code","source":"# Any kind of closing bracket\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Close_Punctuation}', x)).loc[[35, 53, 86]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17f367cfa488218f00209ddce5ec41b741f61979"},"cell_type":"code","source":"# Any kind of opening quote\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Initial_Punctuation}', x)).loc[[903, 986, 1075]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d6f3313b15ec71b099b7c186ff90765ba3359065"},"cell_type":"code","source":"# Any kind of closing quote\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Final_Punctuation}', x)).loc[[173, 260, 317]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c3b7c37c7c8c89262533bd4f193ae580a7a7f874"},"cell_type":"code","source":"# A punctuation character such as an underscore that connects words\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Connector_Punctuation}', x)).loc[[7331, 14208, 21426]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f5b3b03d18ea5eaa578adb473f625d9ab3ff0d2b"},"cell_type":"code","source":"# Any kind of punctuation character that is not a dash, bracket, quote or connector\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Punctuation}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"60d2bcce014734d0243f987e7bd037f34201177f"},"cell_type":"code","source":"# Invisible control characters and unused code points\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other}', x)).loc[[1048, 1655, 4572]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6ac277424d7ce06f5e7e73ce7ea4a0c6926a5955"},"cell_type":"code","source":"# An ASCII or Latin-1 control character: 0x00–0x1F and 0x7F–0x9F\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Control}', x)).loc[[344457, 522266, 566346]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e2e1cdcb8ed9c47e00ca4c77b43e5d3c065f630e"},"cell_type":"code","source":"# Invisible formatting indicator\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Format}', x)).loc[[1048, 1655, 4572]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"becc571b91a509fb3792c7ae5e69dd481800673a"},"cell_type":"code","source":"# Invisible formatting indicator\nX_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Private_Use}', x)).loc[[814877, 1135120, 1192382]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2a1a6dfeb3f1fc6f7046484319e966fec4670ee1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ef2c38b58df9e5b22c84b45b8ed09ade9b0154fe"},"cell_type":"code","source":"train_question_length = X_train['question_length']\ntest_question_length = X_test['question_length']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f1810b03e7f656392f8b72c0f15b76981056bee7"},"cell_type":"code","source":"X_train['prop_category_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter}', x)).str.len() / train_question_length\nX_train['prop_category_lowercase_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Lowercase_Letter}', x)).str.len() / train_question_length\nX_train['prop_category_uppercase_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Uppercase_Letter}', x)).str.len() / train_question_length\nX_train['prop_category_titlecase_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\b\\p{Uppercase_Letter}\\p{Lowercase_Letter}+\\b', x)).str.len() / train_question_length\nX_train['prop_category_cased_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cased_Letter}', x)).str.len() / train_question_length\nX_train['prop_category_modifier_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Letter}', x)).str.len() / train_question_length\nX_train['prop_category_other_letter'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Letter}', x)).str.len() / train_question_length\nX_train['prop_category_mark'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mark}', x)).str.len() / train_question_length\nX_train['prop_category_non_spacing_mark'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Non_Spacing_Mark}', x)).str.len() / train_question_length\nX_train['prop_category_separator'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Separator}', x)).str.len() / train_question_length\nX_train['prop_category_space_separator'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Space_Separator}', x)).str.len() / train_question_length\nX_train['prop_category_symbol'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Symbol}', x)).str.len() / train_question_length\nX_train['prop_category_math_symbol'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Math_Symbol}', x)).str.len() / train_question_length\nX_train['prop_category_currency_symbol'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Currency_Symbol}', x)).str.len() / train_question_length\nX_train['prop_category_modifier_symbol'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Symbol}', x)).str.len() / train_question_length\nX_train['prop_category_other_symbol'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Symbol}', x)).str.len() / train_question_length\nX_train['prop_category_number'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Number}', x)).str.len() / train_question_length\nX_train['prop_category_letter_number'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter_Number}', x)).str.len() / train_question_length\nX_train['prop_category_other_number'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Number}', x)).str.len() / train_question_length\nX_train['prop_category_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_dash_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Dash_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_open_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Open_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_close_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Close_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_initial_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Initial_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_final_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Final_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_connector_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Connector_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_other_punctuation'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Punctuation}', x)).str.len() / train_question_length\nX_train['prop_category_other'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other}', x)).str.len() / train_question_length\nX_train['prop_category_control'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Control}', x)).str.len() / train_question_length\nX_train['prop_category_format'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Format}', x)).str.len() / train_question_length\nX_train['prop_category_format'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Private_Use}', x)).str.len() / train_question_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fea025713f476ac87c87b93e735d5e112970e7ab"},"cell_type":"code","source":"X_test['prop_category_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter}', x)).str.len() / test_question_length\nX_test['prop_category_lowercase_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Lowercase_Letter}', x)).str.len() / test_question_length\nX_test['prop_category_uppercase_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Uppercase_Letter}', x)).str.len() / test_question_length\nX_test['prop_category_titlecase_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\b\\p{Uppercase_Letter}\\p{Lowercase_Letter}+\\b', x)).str.len() / test_question_length\nX_test['prop_category_cased_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cased_Letter}', x)).str.len() / test_question_length\nX_test['prop_category_modifier_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Letter}', x)).str.len() / test_question_length\nX_test['prop_category_other_letter'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Letter}', x)).str.len() / test_question_length\nX_test['prop_category_mark'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mark}', x)).str.len() / test_question_length\nX_test['prop_category_non_spacing_mark'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Non_Spacing_Mark}', x)).str.len() / test_question_length\nX_test['prop_category_separator'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Separator}', x)).str.len() / test_question_length\nX_test['prop_category_space_separator'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Space_Separator}', x)).str.len() / test_question_length\nX_test['prop_category_symbol'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Symbol}', x)).str.len() / test_question_length\nX_test['prop_category_math_symbol'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Math_Symbol}', x)).str.len() / test_question_length\nX_test['prop_category_currency_symbol'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Currency_Symbol}', x)).str.len() / test_question_length\nX_test['prop_category_modifier_symbol'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Modifier_Symbol}', x)).str.len() / test_question_length\nX_test['prop_category_other_symbol'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Symbol}', x)).str.len() / test_question_length\nX_test['prop_category_number'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Number}', x)).str.len() / test_question_length\nX_test['prop_category_letter_number'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Letter_Number}', x)).str.len() / test_question_length\nX_test['prop_category_other_number'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Number}', x)).str.len() / test_question_length\nX_test['prop_category_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_dash_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Dash_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_open_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Open_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_close_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Close_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_initial_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Initial_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_final_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Final_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_connector_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Connector_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_other_punctuation'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other_Punctuation}', x)).str.len() / test_question_length\nX_test['prop_category_other'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Other}', x)).str.len() / test_question_length\nX_test['prop_category_control'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Control}', x)).str.len() / test_question_length\nX_test['prop_category_format'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Format}', x)).str.len() / test_question_length\nX_test['prop_category_format'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Private_Use}', x)).str.len() / test_question_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"793a9d01cca75dc0dffb6aacdb361e0c3d5351f5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87ac1824f8c594dc822354e290c46f3f1d1453e5"},"cell_type":"markdown","source":"Unicode Scripts"},{"metadata":{"trusted":false,"_uuid":"021078904ec93b4e6cba046a4c1736dd4dba24fd"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Common}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0250ab95dacbf39bbfcc9a0d1e357ad80973e6ae"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Arabic}', x)).loc[[3135, 24545, 46644]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8dba8dbc48fbb4fa244fd8e2fab936d7b9eeb413"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Armenian}', x)).loc[[157715, 706999]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2ac1b45aaa9e31a8c933c61212b9950161b70675"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Bengali}', x)).loc[[83908, 226715, 511566]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5e82ad9c11ff11223d92edd475efae5de4177dbf"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Canadian_Aboriginal}', x)).loc[[919693]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0c22429dd3d14bcd0e49dcdd2c5dc38150b0db4f"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cyrillic}', x)).loc[[19906, 22628, 37574]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5fb40ba15a88a47d38d63d72f77f4be69826d302"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Devanagari}', x)).loc[[905, 2194, 26412]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4f30e76b82a3c4b1e8a9a6ddbd71313825d29c79"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Ethiopic}', x)).loc[[236161, 936861]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"65a004ed1e5cc3446c47ac39c26033279764e0a1"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Greek}', x)).loc[[7570, 12077, 13683]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e9c8b019f96b5f8ca4e9071721199d0b7eb36069"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gujarati}', x)).loc[87519]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"37adb031adba338c36a8f1a1ce41b31e1670a8c0"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gurmukhi}', x)).loc[[286009, 518150, 672715]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"129d031356ebb0d02daa35f9156fc377b1336034"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Han}', x)).loc[[1360, 9457, 15165]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1d238a4875447d1765e514a59c95a7d4ab4a00c8"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hangul}', x)).loc[[56312, 134870, 138183]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"21a83748766f423a4bfa8b3cf7f4f85ec3f81e85"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hebrew}', x)).loc[[97812, 259482, 345541]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6807482b14032ba6fd75c0e0192521bec2282f3a"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hiragana}', x)).loc[[18235, 101614, 109595]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9e4f4cdf807ab81fb53dfb0c76981b34691d7049"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Inherited}', x)).loc[[441, 9479, 19836]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ead9904c8a9c0c0fca220141bf943dcd2e7bc34d"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Kannada}', x)).loc[[232429, 1002928]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cb1b56be1ca98effb27af6b60d40aba3e280b3ba"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Katakana}', x)).loc[[81490, 109595, 219625]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"725140e36f031aaef6aed339a40d5f803928060c"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Khmer}', x)).loc[[391253]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a4a8acb77c571074a0fe2a5ad8e2d319d9f4f33a"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Latin}', x)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2fd9503a430be51701c19c544c7e9102dbcb899b"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Malayalam}', x)).loc[[633873]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bea27cff3b970e3ad8126c380abed6e9d25688f4"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mongolian}', x)).loc[[163714]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"97bcdb6d159ef470f94844497fd4e28b535a3e99"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Myanmar}', x)).loc[[619484]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a95053e2680bc034b2c3cf716b6c33077d951f36"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Oriya}', x)).loc[[468196, 769870]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f27415ba24e4e74de0d03e76ffb01bba95daebd5"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Tamil}', x)).loc[[46721, 148815, 262266]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d4a1dd05dd7495a1a247fbda6f6592f6bc51714f"},"cell_type":"code","source":"X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Thai}', x)).loc[[41733, 403251, 782676]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b7a8ba3101df6f327b902bc251dade71147f7ff"},"cell_type":"raw","source":""},{"metadata":{"trusted":false,"_uuid":"25a02169f70801dc30867e2b4b69497301b7e2d2"},"cell_type":"code","source":"X_train['prop_script_common'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Common}', x)).str.len() / train_question_length\nX_train['prop_script_arabic'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Arabic}', x)).str.len() / train_question_length\nX_train['prop_script_armenian'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Armenian}', x)).str.len() / train_question_length\nX_train['prop_script_bengali'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Bengali}', x)).str.len() / train_question_length\nX_train['prop_script_canadian_aboriginal'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Canadian_Aboriginal}', x)).str.len() / train_question_length\nX_train['prop_script_cyrillic'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cyrillic}', x)).str.len() / train_question_length\nX_train['prop_script_devanagari'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Devanagari}', x)).str.len() / train_question_length\nX_train['prop_script_ethiopic'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Ethiopic}', x)).str.len() / train_question_length\nX_train['prop_script_greek'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Greek}', x)).str.len() / train_question_length\nX_train['prop_script_gujarati'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gujarati}', x)).str.len() / train_question_length\nX_train['prop_script_gurmukhi'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gurmukhi}', x)).str.len() / train_question_length\nX_train['prop_script_han'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Han}', x)).str.len() / train_question_length\nX_train['prop_script_hangul'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hangul}', x)).str.len() / train_question_length\nX_train['prop_script_hebrew'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hebrew}', x)).str.len() / train_question_length\nX_train['prop_script_hiragana'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hiragana}', x)).str.len() / train_question_length\nX_train['prop_script_inherited'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Inherited}', x)).str.len() / train_question_length\nX_train['prop_script_kannada'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Kannada}', x)).str.len() / train_question_length\nX_train['prop_script_katakana'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Katakana}', x)).str.len() / train_question_length\nX_train['prop_script_khmer'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Khmer}', x)).str.len() / train_question_length\nX_train['prop_script_latin'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Latin}', x)).str.len() / train_question_length\nX_train['prop_script_malayalam'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Malayalam}', x)).str.len() / train_question_length\nX_train['prop_script_mongolian'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mongolian}', x)).str.len() / train_question_length\nX_train['prop_script_myanmar'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Myanmar}', x)).str.len() / train_question_length\nX_train['prop_script_oriya'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Oriya}', x)).str.len() / train_question_length\nX_train['prop_script_tamil'] = X_train['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Tamil}', x)).str.len() / train_question_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fcc330533059440acadf376cb23802b75c6a553e"},"cell_type":"code","source":"X_test['prop_script_common'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Common}', x)).str.len() / test_question_length\nX_test['prop_script_arabic'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Arabic}', x)).str.len() / test_question_length\nX_test['prop_script_armenian'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Armenian}', x)).str.len() / test_question_length\nX_test['prop_script_bengali'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Bengali}', x)).str.len() / test_question_length\nX_test['prop_script_canadian_aboriginal'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Canadian_Aboriginal}', x)).str.len() / test_question_length\nX_test['prop_script_cyrillic'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Cyrillic}', x)).str.len() / test_question_length\nX_test['prop_script_devanagari'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Devanagari}', x)).str.len() / test_question_length\nX_test['prop_script_ethiopic'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Ethiopic}', x)).str.len() / test_question_length\nX_test['prop_script_greek'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Greek}', x)).str.len() / test_question_length\nX_test['prop_script_gujarati'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gujarati}', x)).str.len() / test_question_length\nX_test['prop_script_gurmukhi'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Gurmukhi}', x)).str.len() / test_question_length\nX_test['prop_script_han'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Han}', x)).str.len() / test_question_length\nX_test['prop_script_hangul'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hangul}', x)).str.len() / test_question_length\nX_test['prop_script_hebrew'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hebrew}', x)).str.len() / test_question_length\nX_test['prop_script_hiragana'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Hiragana}', x)).str.len() / test_question_length\nX_test['prop_script_inherited'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Inherited}', x)).str.len() / test_question_length\nX_test['prop_script_kannada'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Kannada}', x)).str.len() / test_question_length\nX_test['prop_script_katakana'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Katakana}', x)).str.len() / test_question_length\nX_test['prop_script_khmer'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Khmer}', x)).str.len() / test_question_length\nX_test['prop_script_latin'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Latin}', x)).str.len() / test_question_length\nX_test['prop_script_malayalam'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Malayalam}', x)).str.len() / test_question_length\nX_test['prop_script_mongolian'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Mongolian}', x)).str.len() / test_question_length\nX_test['prop_script_myanmar'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Myanmar}', x)).str.len() / test_question_length\nX_test['prop_script_oriya'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Oriya}', x)).str.len() / test_question_length\nX_test['prop_script_tamil'] = X_test['question_text'].progress_apply(lambda x: regex.findall(r'\\p{Tamil}', x)).str.len() / test_question_length","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"163c805aa32945455dcc5030adb5f02f6f41a141"},"cell_type":"raw","source":""},{"metadata":{"trusted":false,"_uuid":"7c738898834b19947c65d402639877c891f16cdc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"df7cfe321d7bf21df693258be7a819d89bd36a85"},"cell_type":"code","source":"train_binary_categories_df = (X_train.filter(like='prop_category') > 0).astype('int').astype('category')\ntrain_binary_categories_df.columns = train_binary_categories_df.columns.str.replace(r'prop_', 'is_')\ntrain_binary_scripts_df = (X_train.filter(like='prop_script') > 0).astype('int').astype('category')\ntrain_binary_scripts_df.columns = train_binary_scripts_df.columns.str.replace(r'prop_', 'is_')\n\nX_train = pd.merge(X_train, train_binary_categories_df, left_index=True, right_index=True)\nX_train = pd.merge(X_train, train_binary_scripts_df, left_index=True, right_index=True)\nX_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"69d664d250cdb05e6edd1e55805a7d159b2b6d05"},"cell_type":"code","source":"test_binary_categories_df = (X_test.filter(like='prop_category') > 0).astype('int').astype('category')\ntest_binary_categories_df.columns = test_binary_categories_df.columns.str.replace(r'prop_', 'is_')\ntest_binary_scripts_df = (X_test.filter(like='prop_script') > 0).astype('int').astype('category')\ntest_binary_scripts_df.columns = test_binary_scripts_df.columns.str.replace(r'prop_', 'is_')\n\nX_test = pd.merge(X_test, test_binary_categories_df, left_index=True, right_index=True)\nX_test = pd.merge(X_test, test_binary_scripts_df, left_index=True, right_index=True)\nX_test.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4bf1b1a674390ffe8c601b702792ad3807b62838"},"cell_type":"raw","source":""},{"metadata":{"_uuid":"7e2422dd694ab816fa253bc5c71ce4fa8bd87b2f"},"cell_type":"markdown","source":"**Unicde categories/scripts statistics**"},{"metadata":{"_uuid":"a33674a5a4854e16768e2d6faba957444874b256"},"cell_type":"markdown","source":"After processing in the previous section, we have 2 groups of features:\n- Binary features, which indicate whether the characters of specific category/script is in question\n- Float features - proportion of the character of specific category/script relative to the length of this question\n\nTherefore using these features, we can answer the questions like the how many documents in train/test datasets or bad/good questions contains the character of specific category/script"},{"metadata":{"trusted":false,"_uuid":"948d344c34a1516f6f6de3895c7420c951b868f4"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True, sharey=True)\nfig.suptitle('Train/Test mean unicode categories/scripts proportion distribution (absolute values)', y=1.05, fontsize=14)\n\nX_train.filter(like='prop_category').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_test.filter(like='prop_category').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Proportion (log10)')\naxes[1].set_title('Test - Unicode Categories Proportion (log10)')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True, sharey=True)\nX_train.filter(like='prop_script').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_test.filter(like='prop_script').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Proportion (log10)')\naxes[1].set_title('Test - Unicode Scripts Proportion (log10)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fb06a93ecaaacd6fc6188ea4dd91cff9853cef15"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True, sharey=True)\nfig.suptitle('Train/Test mean unicode categories/scripts boolean distribution (absolute values)', y=1.05, fontsize=14)\n\nX_train.filter(like='is_category').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_test.filter(like='is_category').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Boolean (log10)')\naxes[1].set_title('Test - Unicode Categories Boolean (log10)')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True, sharey=True)\nX_train.filter(like='is_script').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_test.filter(like='is_script').mean().sort_values(ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Boolean (log10)')\naxes[1].set_title('Test - Unicode Scripts Boolean (log10)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7f32572976c1ea0f448c1dd52753173fc033eef7"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True)\nfig.suptitle('Train mean unicode categories/scripts proportion/boolean distribution by target (absolute values)', y=1.05, fontsize=14)\n\nX_train.filter(like='prop_category').groupby(X_train['target']).mean().T.sort_values(1, ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_train.filter(like='is_category').astype('int').groupby(X_train['target']).mean().T.sort_values(1, ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Proportion (log10)')\naxes[1].set_title('Train - Unicode Categories Boolean (log10)')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True)\nX_train.filter(like='prop_script').groupby(X_train['target']).mean().T.sort_values(1, ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[0])\nX_train.filter(like='is_script').astype('int').groupby(X_train['target']).mean().T.sort_values(1, ascending=False)[::-1].apply(np.log10).plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Proportion (log10)')\naxes[1].set_title('Train - Unicode Scripts Boolean (log10))')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"42bc47553d1cb641d0f7258289d035d4c07cb9d7"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True)\nfig.suptitle('Train mean unicode categories/scripts proportion/boolean distribution by target (relative values)', y=1.05, fontsize=14)\n\nX_train.filter(like='prop_category').groupby(X_train['target']).mean().apply(lambda x: x / x.sum()).T.sort_values(1, ascending=False)[::-1].plot.barh(ax=axes[0])\nX_train.filter(like='is_category').astype('int').groupby(X_train['target']).mean().apply(lambda x: x / x.sum()).T.sort_values(1, ascending=False)[::-1].plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Proportion')\naxes[1].set_title('Train - Unicode Categories Boolean')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6), sharex=True)\nX_train.filter(like='prop_script').groupby(X_train['target']).mean().apply(lambda x: x / x.sum()).T.sort_values(1, ascending=False)[::-1].plot.barh(ax=axes[0])\nX_train.filter(like='is_script').astype('int').groupby(X_train['target']).mean().apply(lambda x: x / x.sum()).T.sort_values(1, ascending=False)[::-1].plot.barh(ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Proportion')\naxes[1].set_title('Train - Unicode Scripts Boolean')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cd6693d76228fd6bb289dc0813d6b07abdeb0bdc"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20, 8), sharex=True)\nfig.suptitle('Train/test correlation of unicode categories/scripts proportion features', y=1.05, fontsize=14)\n\nsns.heatmap(X_train.filter(like='prop_category').corr(), ax=axes[0])\nsns.heatmap(X_test.filter(like='prop_category').corr(), ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Proportion')\naxes[1].set_title('Test - Unicode Categories Proportion')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(20, 8), sharex=True)\nsns.heatmap(X_train.filter(like='prop_script').corr(), ax=axes[0])\nsns.heatmap(X_test.filter(like='prop_script').corr(), ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Proportion')\naxes[1].set_title('Test - Unicode Scripts Proportion')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0b9d1063737b4ad634da5c8ad0142e644397de1f"},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20, 8), sharex=True)\nfig.suptitle('Train correlation of unicode categories/scripts proportion by target', y=1.05, fontsize=14)\n\nsns.heatmap(X_train.filter(like='prop_category').groupby(X_train['target']).corr().loc[0], ax=axes[0])\nsns.heatmap(X_train.filter(like='prop_category').groupby(X_train['target']).corr().loc[1], ax=axes[1])\naxes[0].set_title('Train - Unicode Categories Proportion (Good)')\naxes[1].set_title('Train - Unicode Categories Proportion (Bad)')\nplt.tight_layout()\n\nfig, axes = plt.subplots(1, 2, figsize=(20, 8), sharex=True)\nsns.heatmap(X_train.filter(like='prop_script').groupby(X_train['target']).corr().loc[0], ax=axes[0])\nsns.heatmap(X_train.filter(like='prop_script').groupby(X_train['target']).corr().loc[1], ax=axes[1])\naxes[0].set_title('Train - Unicode Scripts Proportion (Good)')\naxes[1].set_title('Train - Unicode Scripts Proportion (Bad)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e5cbab60516a6a08e7d83e5f303644161718e7e"},"cell_type":"markdown","source":"This is my first public kernel, and any ideas/questions/suggestions are appreciated."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}