{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q demoji","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nfrom joblib import Parallel, delayed\nimport time\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import ChainMap, Counter\nimport itertools\n\nfrom wordcloud import WordCloud\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import emoji\nimport demoji\ndemoji.download_codes()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take a look at the emoji contained in one of the training datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"path_data = '../input/jigsaw-multilingual-toxic-comment-classification/'\ncols_to_use = ['id', 'comment_text', 'toxic']\n\ntrain_tc = pd.read_csv(path_data + 'jigsaw-toxic-comment-train.csv', usecols=cols_to_use)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# encoding id to save memory\ntrain_tc.id = LabelEncoder().fit_transform(train_tc.id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tc.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_emoji(idx, row):\n    if demoji.findall(row):\n        return { idx: [v for _, v in demoji.findall(row).items()] }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nparallel =  Parallel(n_jobs=-1, backend='multiprocessing', verbose=0)\njoblist = [ delayed(find_emoji)(idx, row) for idx, row in zip(train_tc.id.values, \n                                                              train_tc.comment_text.values) if delayed(find_emoji)(idx, row) ]\nretlist  =  parallel( joblist )\nlist_of_dicts = list(filter(lambda x: type(x)==dict, retlist))\nd = dict(ChainMap(*list_of_dicts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tc['emoji_cnt'] = train_tc.id.map( {k:len(v) for k, v in d.items()} ).fillna(0).astype(int)\ntrain_tc['emoji'] = train_tc.id.map(d).fillna('none')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot( train_tc.emoji_cnt[train_tc.emoji_cnt != 0], hue=train_tc.toxic)\nplt.title(\"Number of emojis in comment text\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tc['emoji_cnt'] = train_tc[train_tc.emoji != 'none'].emoji.apply( lambda x: len(x) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, ax = plt.subplots(figsize=(15,5)) \nax = sns.heatmap( pd.crosstab(train_tc.emoji_cnt, train_tc.toxic), cmap=\"Blues\", fmt='g', annot=True, cbar=False )\nplt.title(\"Number of emojis in comment text\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_toxic_emoji = list(itertools.chain(*train_tc[(train_tc.emoji != 'none') & (train_tc.toxic == 0)].emoji.values))\ntoxic_emoji = list(itertools.chain(*train_tc[(train_tc.emoji != 'none') & (train_tc.toxic == 1)].emoji.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c0 = Counter()\nc1 = Counter()\nfor emoji_non_toxic, emoji_toxic in zip(non_toxic_emoji, toxic_emoji):\n    c0[emoji_non_toxic] += 1\n    c1[emoji_toxic] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Most common non toxic:\", c0.most_common(5))\nprint()\nprint(\"Most common toxic:\", c1.most_common(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def concat_items(lst):\n    return list(map(lambda x: x.replace(' ', '_'), lst))\n\ntext0 = ' '.join(concat_items(non_toxic_emoji))\ntext1 = ' '.join(concat_items(toxic_emoji))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_wordcloud(text, title, toxic=True):\n    \n    back_color = \"grey\" if toxic else \"white\"\n    wordcloud = WordCloud(relative_scaling=1.0, width=1000, \n                          height=1000, max_font_size=100, \n                          max_words=100, background_color=back_color).generate(text)\n    plt.figure(figsize=(15,10))\n    plt.imshow(wordcloud, interpolation=\"bilinear\")\n    plt.title(title, fontsize= 30)\n    plt.axis(\"off\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(text=text1, title=\"Toxic emojis\", toxic=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(text=text0, title=\"Non toxic emojis\", toxic=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Based on the foregoing, it's obvious that сharacters defined as emoji do not contain meaning in terms of the problem being solved."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}