{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this project I develop the following strategy to predict toxicity of multi language comments.17368 comments from training set were selected and translated on six different languages using translators library. For toxic anf normal comments voacabulries were created separately and after were combained. The model contain embedding layer, two layers with GRU cells and one Dense layer.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import sys\nimport sklearn\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport os\nimport pandas as pd \n\n# to make this notebook's output stable across runs\nnp.random.seed(42)\n\n# To plot pretty figures\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rc('axes', labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\n\n# Ignore useless warnings (see SciPy issue #5998)\nimport warnings\nwarnings.filterwarnings(action=\"ignore\", message=\"^internal gelsd\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TRAIN DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading data\ninput_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\ndef load_csv_data(input_dir, filename):\n    csv_path = input_dir + filename + \".csv\"\n    return pd.read_csv(csv_path)\n\ntrain_data_b01 = load_csv_data(input_dir, \"jigsaw-toxic-comment-train-processed-seqlen128\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct toxic label in trainset\n\ntrain_data_b01['total_toxicity'] = 0\ns = train_data_b01.shape[0]\n\nfor i in range(s):\n    counter = 0\n    if train_data_b01[\"toxic\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"toxic\"][i]\n        \n    if train_data_b01[\"severe_toxic\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"severe_toxic\"][i]\n        \n    if train_data_b01[\"obscene\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"obscene\"][i]\n        \n    if train_data_b01[\"threat\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"threat\"][i]\n        \n    if train_data_b01[\"insult\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"insult\"][i]\n    \n    if train_data_b01[\"identity_hate\"][i] > 0:\n        counter += 1\n        train_data_b01['total_toxicity'][i] += train_data_b01[\"identity_hate\"][i]\n        \n    if counter > 0 :\n        train_data_b01['total_toxicity'][i] = train_data_b01['total_toxicity'][i]/counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct trainset\n\ndata = { 'comment_text' : (train_data_b01['comment_text']),\n        'toxic' : train_data_b01['total_toxicity']   \n}\n\ntrain_data_mod = pd.DataFrame(data, columns = ['comment_text', 'toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# separate normal and toxic comments\n\nmax_comment_size = 300\n\n# remove emoji from comments\nimport emoji\ndef give_emoji_free_text(text):\n    return emoji.get_emoji_regexp().sub(r'', text)\n\ncomment_normal = []\ncomment_toxic = []\nsize = train_data_mod.shape[0]\n\nwith tf.device('/CPU:0'):\n    for i in range(size):\n        comment = train_data_mod['comment_text'][i]\n        comment = give_emoji_free_text(comment)\n        comment = comment[0:max_comment_size].replace(\"<r\\s*/?>\", \" \").replace(\"[^a-zA-Z']\", \" \")\n        if train_data_mod['toxic'][i] == 0:\n            comment_normal.append(comment)\n        else:\n            comment_toxic.append(comment)\n        \nnormar_labels = np.zeros(len(comment_normal))\ntoxic_labels = np.ones(len(comment_toxic))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# find duplicates\n\nimport collections\nprint([count for item, count in collections.Counter(comment_normal).items() if count > 1])\nprint([count for item, count in collections.Counter(comment_toxic).items() if count > 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete duplicates\n\ncomment_normal = list(dict.fromkeys(comment_normal))\ncomment_toxic = list(dict.fromkeys(comment_toxic))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check it\nprint([count for item, count in collections.Counter(comment_normal).items() if count > 1])\nprint([count for item, count in collections.Counter(comment_toxic).items() if count > 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function for combining  text with separators from comment data\n\ndef text_packet(comment_normal, comment_toxic, inds_n, inds_t, packet_size =25, separator = \"######\"):\n    normal_comments = []\n    toxic_comments = []\n    \n    for i in range(packet_size):\n        n_ind = inds_n.pop(i)\n        t_ind = inds_t.pop(i)\n        normal_comments.append(comment_normal[n_ind])\n        toxic_comments.append(comment_toxic[t_ind])\n        \n    text_normal = ''\n    text_toxic = ''\n    \n    for comment in normal_comments:\n        text_normal += comment + separator\n    \n    for comment in toxic_comments:\n        text_toxic += comment + separator\n        \n    return text_normal, text_toxic  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# back from text to list of comments\n\ndef text_to_list(text, separator = \"######\"):\n    text_comments = []\n    text_comments = text.split(separator)\n    try:\n        text_comments.remove('')\n    except:\n        text_comments = text_comments\n    return text_comments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# append prepared lists of comments, labels, langs\n\ndef append_data_lists(text, toxic_list, lang_list, comment_list, toxic = 1, lang = 'tr'):\n    text_list = text_to_list(text, separator = \"######\")\n    n = len(text_list) \n    for i in range(n):\n        toxic_list.append(toxic)\n        lang_list.append(lang)\n        comment_list.append(text_list[i])\n    return toxic_list, lang_list, comment_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare random indecex for data selection \ndef random_inds(size):\n    inds = list(np.random.permutation(size))\n    return inds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we will try to translate some train data on six different languages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# translate english comments\n\nnp.random.seed(42)\n\nlen_n = len(comment_normal)\nlen_t = len(comment_toxic)\n\ninds_n_tr = random_inds(len_n)\ninds_t_tr = random_inds(len_t)\n\ninds_n_pt = random_inds(len_n)\ninds_t_pt = random_inds(len_t)\n\ninds_n_ru = random_inds(len_n)\ninds_t_ru = random_inds(len_t)\n\ninds_n_fr = random_inds(len_n)\ninds_t_fr = random_inds(len_t)\n\ninds_n_it = random_inds(len_n)\ninds_t_it = random_inds(len_t)\n\ninds_n_es = random_inds(len_n)\ninds_t_es = random_inds(len_t)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For translation we will use translators library and google API","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#pip install translators","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    import time\n    import translators as ts\n    toxic_list = []\n    lang_list = []\n    comment_list = []\n    API = ts.google\n    packet_size = 25\n    sleep_time = 2\n    num_iter = 24\n    \n    for iteration in range(num_iter):\n        import translators as ts    \n        print(\"Iteration: \" + str(iteration)) \n       \n        lang = 'tr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_tr, inds_t_tr, \n                                     packet_size = packet_size)\n        text_t_tr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_tr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'pt'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_pt, inds_t_pt, \n                                     packet_size = packet_size)\n        text_t_pt = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_pt, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_pt = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_pt, \n                                                            toxic_list, lang_list, comment_list,\n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n        lang = 'ru'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_ru, inds_t_ru,\n                                    packet_size = packet_size)\n        text_t_ru = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_ru, \n                                                            toxic_list, lang_list, comment_list,                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_ru = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_ru, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n    \n        lang = 'fr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_fr, inds_t_fr, \n                                     packet_size = packet_size)\n        text_t_fr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_fr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'it'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_it, inds_t_it, \n                                     packet_size = packet_size)\n        text_t_it = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_it = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'es'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_es, inds_t_es, \n                                     packet_size = packet_size)\n        text_t_es = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_es = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n        \nexcept:\n    print(\"Error\")\n       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The result of translation avalible in the file mini_multilang_trainset_1.csv in the '/kaggle/working'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmini_multilang_trainset_1 = pd.DataFrame(columns = ['comment', 'toxic', 'lang'])\nmini_multilang_trainset_1['comment'] = comment_list\nmini_multilang_trainset_1['toxic'] = toxic_list\nmini_multilang_trainset_1['lang'] = lang_list\n\nmini_multilang_trainset_1.to_csv('mini_multilang_trainset_1.csv',index = False)\n\"\"\"\npreprocessed_dir = '/kaggle/input/preprocessed-data/'\noutput_dir = '/kaggle/working/'\n\nmini_multilang_trainset_1 = load_csv_data(preprocessed_dir, \"mini_multilang_trainset_1\")\nmini_multilang_trainset_1.to_csv(output_dir + 'mini_multilang_trainset_1.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now it is neccessary to save unused indeces inds_n_tr, inds_t_tr, inds_n_pt, inds_t_pt, inds_n_ru, inds_t_ru, inds_n_fr, inds_t_fr, inds_n_it, inds_t_it, inds_n_es, inds_t_es. The Google server block IP after some limit and we can continue process of translation only on next day.\n\nDataFrames with unused undeces are avalible in the files unused_normal_indeces_after_step_1.csv and unused_toxic_indeces_after_step_1.csvin the '/kaggle/working' directory.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nunused_normal_indeces_1 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_normal_indeces_1['ind_tr'] = inds_n_tr\nunused_normal_indeces_1['ind_pt'] = inds_n_pt\nunused_normal_indeces_1['ind_ru'] = inds_n_ru\nunused_normal_indeces_1['ind_fr'] = inds_n_fr\nunused_normal_indeces_1['ind_it'] = inds_n_it\nunused_normal_indeces_1['ind_es'] = inds_n_es\n\nunused_toxic_indeces_1 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_toxic_indeces_1['ind_tr'] = inds_t_tr\nunused_toxic_indeces_1['ind_pt'] = inds_t_pt\nunused_toxic_indeces_1['ind_ru'] = inds_t_ru\nunused_toxic_indeces_1['ind_fr'] = inds_t_fr\nunused_toxic_indeces_1['ind_it'] = inds_t_it\nunused_toxic_indeces_1['ind_es'] = inds_t_es\n\nunused_normal_indeces_1.to_csv('unused_normal_indeces_after_step_1',index = False)\nunused_toxic_indeces_1.to_csv('unused_toxic_indeces_after_step_1.csv',index = False)\n\"\"\"\n\nunused_normal_indeces_after_step_1 = load_csv_data(preprocessed_dir, \"unused_normal_indeces_after_step_1\")\nunused_normal_indeces_after_step_1.to_csv(output_dir + 'unused_normal_indeces_after_step_1.csv', index = False)\n\nunused_toxic_indeces_after_step_1 = load_csv_data(preprocessed_dir, \"unused_toxic_indeces_after_step_1\")\nunused_toxic_indeces_after_step_1.to_csv(output_dir + 'unused_toxic_indeces_after_step_1.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare random indeces from step 1\n\ninds_n_tr = list(unused_normal_indeces_after_step_1['ind_tr'])\ninds_t_tr = list(unused_toxic_indeces_after_step_1['ind_tr'])\n\ninds_n_pt = list(unused_normal_indeces_after_step_1['ind_pt'])\ninds_t_pt = list(unused_toxic_indeces_after_step_1['ind_pt'])\n\ninds_n_ru = list(unused_normal_indeces_after_step_1['ind_ru'])\ninds_t_ru = list(unused_toxic_indeces_after_step_1['ind_ru'])\n\ninds_n_fr = list(unused_normal_indeces_after_step_1['ind_fr'])\ninds_t_fr = list(unused_toxic_indeces_after_step_1['ind_fr'])\n\ninds_n_it = list(unused_normal_indeces_after_step_1['ind_it'])\ninds_t_it = list(unused_toxic_indeces_after_step_1['ind_it'])\n\ninds_n_es = list(unused_normal_indeces_after_step_1['ind_es'])\ninds_t_es = list(unused_toxic_indeces_after_step_1['ind_es'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    import time\n    import translators as ts\n    toxic_list = []\n    lang_list = []\n    comment_list = []\n    API = ts.google\n    packet_size = 25\n    sleep_time = 2\n    num_iter = 11\n    \n    for iteration in range(num_iter):\n        import translators as ts    \n        print(\"Iteration: \" + str(iteration)) \n       \n        lang = 'tr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_tr, inds_t_tr, \n                                     packet_size = packet_size)\n        text_t_tr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_tr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'pt'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_pt, inds_t_pt, \n                                     packet_size = packet_size)\n        text_t_pt = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_pt, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_pt = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_pt, \n                                                            toxic_list, lang_list, comment_list,\n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n        lang = 'ru'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_ru, inds_t_ru,\n                                    packet_size = packet_size)\n        text_t_ru = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_ru, \n                                                            toxic_list, lang_list, comment_list,                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_ru = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_ru, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n    \n        lang = 'fr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_fr, inds_t_fr, \n                                     packet_size = packet_size)\n        text_t_fr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_fr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'it'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_it, inds_t_it, \n                                     packet_size = packet_size)\n        text_t_it = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_it = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'es'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_es, inds_t_es, \n                                     packet_size = packet_size)\n        text_t_es = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_es = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n        \nexcept:\n    print(\"Error\")\n       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The result of translation avalible in the file mini_multilang_trainset_2.csv in the '/kaggle/working'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmini_multilang_trainset_2 = pd.DataFrame(columns = ['comment', 'toxic', 'lang'])\nmini_multilang_trainset_2['comment'] = comment_list\nmini_multilang_trainset_2['toxic'] = toxic_list\nmini_multilang_trainset_2['lang'] = lang_list\n\nmini_multilang_trainset_2.to_csv('mini_multilang_trainset_2.csv',index = False)\n\"\"\"\nmini_multilang_trainset_2 = load_csv_data(preprocessed_dir, \"mini_multilang_trainset_2\")\nmini_multilang_trainset_2.to_csv(output_dir + 'mini_multilang_trainset_2.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now it is neccessary to save unused indeces inds_n_tr, inds_t_tr, inds_n_pt, inds_t_pt, inds_n_ru, inds_t_ru, inds_n_fr, inds_t_fr, inds_n_it, inds_t_it, inds_n_es, inds_t_es. \n\nDataFrames with unused undeces are avalible in the files unused_normal_indeces_after_step_2.csv and unused_toxic_indeces_after_step_2.csvin the '/kaggle/working' directory.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nunused_normal_indeces_2 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_normal_indeces_2['ind_tr'] = inds_n_tr\nunused_normal_indeces_2['ind_pt'] = inds_n_pt\nunused_normal_indeces_2['ind_ru'] = inds_n_ru\nunused_normal_indeces_2['ind_fr'] = inds_n_fr\nunused_normal_indeces_2['ind_it'] = inds_n_it\nunused_normal_indeces_2['ind_es'] = inds_n_es\n\nunused_toxic_indeces_2 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_toxic_indeces_2['ind_tr'] = inds_t_tr\nunused_toxic_indeces_2['ind_pt'] = inds_t_pt\nunused_toxic_indeces_2['ind_ru'] = inds_t_ru\nunused_toxic_indeces_2['ind_fr'] = inds_t_fr\nunused_toxic_indeces_2['ind_it'] = inds_t_it\nunused_toxic_indeces_2['ind_es'] = inds_t_es\n\nunused_normal_indeces_2.to_csv('unused_normal_indeces_2.csv',index = False)\nunused_toxic_indeces_2.to_csv('unused_toxic_indeces_2.csv',index = False)\n\"\"\"\n\nunused_normal_indeces_after_step_2 = load_csv_data(preprocessed_dir, \"unused_normal_indeces_after_step_2\")\nunused_normal_indeces_after_step_2.to_csv(output_dir + 'unused_normal_indeces_after_step_2.csv', index = False)\n\nunused_toxic_indeces_after_step_2 = load_csv_data(preprocessed_dir, \"unused_toxic_indeces_after_step_2\")\nunused_toxic_indeces_after_step_2.to_csv(output_dir + 'unused_toxic_indeces_after_step_2.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare random indeces from step 2\n\ninds_n_tr = list(unused_normal_indeces_after_step_2['ind_tr'])\ninds_t_tr = list(unused_toxic_indeces_after_step_2['ind_tr'])\n\ninds_n_pt = list(unused_normal_indeces_after_step_2['ind_pt'])\ninds_t_pt = list(unused_toxic_indeces_after_step_2['ind_pt'])\n\ninds_n_ru = list(unused_normal_indeces_after_step_2['ind_ru'])\ninds_t_ru = list(unused_toxic_indeces_after_step_2['ind_ru'])\n\ninds_n_fr = list(unused_normal_indeces_after_step_2['ind_fr'])\ninds_t_fr = list(unused_toxic_indeces_after_step_2['ind_fr'])\n\ninds_n_it = list(unused_normal_indeces_after_step_2['ind_it'])\ninds_t_it = list(unused_toxic_indeces_after_step_2['ind_it'])\n\ninds_n_es = list(unused_normal_indeces_after_step_2['ind_es'])\ninds_t_es = list(unused_toxic_indeces_after_step_2['ind_es'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    import time\n    import translators as ts\n    toxic_list = []\n    lang_list = []\n    comment_list = []\n    API = ts.google\n    packet_size = 25\n    sleep_time = 2\n    num_iter = 15\n    \n    for iteration in range(num_iter):\n        import translators as ts    \n        print(\"Iteration: \" + str(iteration)) \n       \n        lang = 'tr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_tr, inds_t_tr, \n                                     packet_size = packet_size)\n        text_t_tr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_tr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_tr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'pt'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_pt, inds_t_pt, \n                                     packet_size = packet_size)\n        text_t_pt = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_pt, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_pt = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_pt, \n                                                            toxic_list, lang_list, comment_list,\n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n        lang = 'ru'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_ru, inds_t_ru,\n                                    packet_size = packet_size)\n        text_t_ru = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_ru, \n                                                            toxic_list, lang_list, comment_list,                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_ru = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_ru, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n       \n    \n        lang = 'fr'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_fr, inds_t_fr, \n                                     packet_size = packet_size)\n        text_t_fr = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_fr = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_fr, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'it'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_it, inds_t_it, \n                                     packet_size = packet_size)\n        text_t_it = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_it = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_it, \n                                                            toxic_list, lang_list, comment_list, \n                                                            toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n    \n        lang = 'es'\n        text_n, text_t = text_packet(comment_normal, comment_toxic, inds_n_es, inds_t_es, \n                                     packet_size = packet_size)\n        text_t_es = API(text_t, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_t_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 1, lang = lang)\n        time.sleep(sleep_time)\n        text_n_es = API(text_n, to_language=lang)\n        toxic_list, lang_list, comment_list = append_data_lists(text_n_es, \n                                                            toxic_list, lang_list, \n                                                            comment_list, toxic = 0, lang = lang)\n        time.sleep(sleep_time*2)\n        \nexcept:\n    print(\"Error\")\n       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The result of translation avalible in the file mini_multilang_trainset_3.csv in the '/kaggle/working'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmini_multilang_trainset_3 = pd.DataFrame(columns = ['comment', 'toxic', 'lang'])\nmini_multilang_trainset_3['comment'] = comment_list\nmini_multilang_trainset_3['toxic'] = toxic_list\nmini_multilang_trainset_3['lang'] = lang_list\n\nmini_multilang_trainset_3.to_csv('mini_multilang_trainset_3.csv',index = False)\n\"\"\"\nmini_multilang_trainset_3 = load_csv_data(preprocessed_dir, \"mini_multilang_trainset_3\")\nmini_multilang_trainset_3.to_csv(output_dir + 'mini_multilang_trainset_3.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now it is neccessary to save unused indeces inds_n_tr, inds_t_tr, inds_n_pt, inds_t_pt, inds_n_ru, inds_t_ru, inds_n_fr, inds_t_fr, inds_n_it, inds_t_it, inds_n_es, inds_t_es.\n\nDataFrames with unused undeces are avalible in the files unused_normal_indeces_after_step_3.csv and unused_toxic_indeces_after_step_3.csvin the '/kaggle/working' directory.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nunused_normal_indeces_3 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_normal_indeces_3['ind_tr'] = inds_n_tr\nunused_normal_indeces_3['ind_pt'] = inds_n_pt\nunused_normal_indeces_3['ind_ru'] = inds_n_ru\nunused_normal_indeces_3['ind_fr'] = inds_n_fr\nunused_normal_indeces_3['ind_it'] = inds_n_it\nunused_normal_indeces_3['ind_es'] = inds_n_es\n\nunused_toxic_indeces_3 = pd.DataFrame(columns = ['ind_tr', 'ind_pt', 'ind_ru', 'ind_fr', 'ind_it', 'ind_es'])\nunused_toxic_indeces_3['ind_tr'] = inds_t_tr\nunused_toxic_indeces_3['ind_pt'] = inds_t_pt\nunused_toxic_indeces_3['ind_ru'] = inds_t_ru\nunused_toxic_indeces_3['ind_fr'] = inds_t_fr\nunused_toxic_indeces_3['ind_it'] = inds_t_it\nunused_toxic_indeces_3['ind_es'] = inds_t_es\n\nunused_normal_indeces_3.to_csv('unused_normal_indeces_3.csv',index = False)\nunused_toxic_indeces_3.to_csv('unused_toxic_indeces_3.csv',index = False)\n\"\"\"\n\nunused_normal_indeces_after_step_3 = load_csv_data(preprocessed_dir, \"unused_normal_indeces_after_step_3\")\nunused_normal_indeces_after_step_3.to_csv(output_dir + 'unused_normal_indeces_after_step_3.csv', index = False)\n\nunused_toxic_indeces_after_step_3 = load_csv_data(preprocessed_dir, \"unused_toxic_indeces_after_step_3\")\nunused_toxic_indeces_after_step_3.to_csv(output_dir + 'unused_toxic_indeces_after_step_3.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will select 2800 english comments ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# add english comments\n\nlen_n = len(comment_normal)\nlen_t = len(comment_toxic)\n\nnum_eng_comments = 2800\n\ninds_n_en =random_inds(len_n)\ninds_t_en =random_inds(len_t)\n\ntoxic_list_en = []\nlang_list_en = []\ncomment_list_en = []\n\nfor i in range(num_eng_comments//2):\n    ind = inds_n_en[i]\n    comment_list_en.append(comment_normal[ind])\n    lang_list_en.append('en')\n    toxic_list_en.append(0)\n    \nfor i in range(num_eng_comments//2):\n    ind = inds_t_en[i]\n    comment_list_en.append(comment_toxic[ind])\n    lang_list_en.append('en')\n    toxic_list_en.append(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mini english dataset avalible in the file 'mini_multilang_trainset_en_s2800.csv' in the directory '/kaggle/working'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmini_multilang_trainset_en = pd.DataFrame(columns = ['comment', 'toxic', 'lang'])\nmini_multilang_trainset_en['comment'] = comment_list_en\nmini_multilang_trainset_en['toxic'] = toxic_list_en\nmini_multilang_trainset_en['lang'] = lang_list_en\n\nmini_multilang_trainset_en.to_csv('mini_multilang_trainset_en_s2800.csv',index = False)\n\"\"\"\nmini_multilang_trainset_en = load_csv_data(preprocessed_dir, \"mini_multilang_trainset_en_s2800\")\nmini_multilang_trainset_en.to_csv(output_dir + 'mini_multilang_trainset_en_s2800.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Combine all mini train sets to one","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"frames = [mini_multilang_trainset_1, mini_multilang_trainset_2, mini_multilang_trainset_3, mini_multilang_trainset_en]\n\nmini_multilang_trainset_combo  = pd.concat(frames)\nmini_multilang_trainset_combo.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can manyally check our dataset using, for example Exele or Numbers and delete some comments-artifacts and empty comments (I spend about 15 min on this quick check). Corrected dataset avalible in the file 'mini_multilang_trainset_combo_manually_corrected.csv' in the directory '/kaggle/working'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mini_multilang_trainset_combo = load_csv_data(preprocessed_dir, \"mini_multilang_trainset_combo_manually_corrected\")\nmini_multilang_trainset_combo.to_csv(output_dir + 'mini_multilang_trainset_combo_manually_corrected.csv', index = False)\n\nmini_multilang_trainset_combo.drop_duplicates(keep=False,inplace=True)\nmini_multilang_trainset_combo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mini_multilang_trainset_combo['lang'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will change all characters to english using translit function,clear text from artifacts and trash and split normal and toxic comments.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\n\ndef clean_text(text):\n    # clear data and time\n    text = re.sub('\\d{2}.\\d{2}.\\d{2}, \\d{2}:\\d{2}:\\d{2}', '', text)\n    text = re.sub('\\d{2}.\\d{2} \\d{2}:\\d{2}', '', text)\n    \n    # remove whitespace before and after word\n    text = re.sub('-\\s\\r\\n\\|-\\s\\r\\n|\\r\\n|[«»]|[\"\"]|[><]|\"[\\[]]|//\"', '', text)\n    text = re.sub('[«»]|[\"\"]|[><]|\"[\\[]]\"', '', text)\n    text = re.sub('[~-¿:;_\"?*!@#$^&%()]|[+=]|[[]|[]]|[/]', ' ', text)\n    \n    text = re.sub(r'\\r\\n\\t|\\n|\\r\\t|\\\\n|&gt', ' ', text)\n    text = re.sub(r'[\\xad]|[\\s+]', ' ', text)\n    text = text.strip().lower()\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def translit(string):\n    \"\"\" This function works just fine \"\"\"\n    capital_letters = {\n\n    }\n\n    lower_case_letters = {\n        u'а': u'a',\n        u'б': u'b',\n        u'в': u'v',\n        u'г': u'g',\n        u'д': u'd',\n        u'е': u'e',\n        u'ё': u'e',\n        u'ж': u'zh',\n        u'з': u'z',\n        u'и': u'i',\n        u'й': u'y',\n        u'к': u'k',\n        u'л': u'l',\n        u'м': u'm',\n        u'н': u'n',\n        u'о': u'o',\n        u'п': u'p',\n        u'р': u'r',\n        u'с': u's',\n        u'т': u't',\n        u'у': u'u',\n        u'ф': u'f',\n        u'х': u'h',\n        u'ц': u'ts',\n        u'ч': u'ch',\n        u'ш': u'sh',\n        u'щ': u'sch',\n        u'ъ': u'',\n        u'ы': u'y',\n        u'ь': u'',\n        u'э': u'e',\n        u'ю': u'yu',\n        u'я': u'ya',\n        \n        u'ö': u'o',\n        u'ü': u'u',\n        u'ş': u's',\n        u'ç': u'c',\n        u'ğ': u'g',\n        u'â': u'a',\n        u'i̇': u'i',\n        \n        u'ó': u'o',\n        u'é': u'e',\n        u'ñ': u'n',\n        u'á': u'a',\n        u'í': u'i',\n        \n        u'ã': u'a',\n        u'ú': u'u',\n        u'ê': u'e',\n        u'à': u'a',\n        u'õ': u'o',\n        u'ĩ': u'i',\n        u'è': u'i',\n    }\n\n    translit_string = \"\"\n\n    for index, char in enumerate(string):\n        if char in lower_case_letters.keys():\n            char = lower_case_letters[char]\n        elif char in capital_letters.keys():\n            char = capital_letters[char]\n            if len(string) > index+1:\n                if string[index+1] not in lower_case_letters.keys():\n                    char = char.upper()\n            else:\n                char = char.upper()\n        translit_string += char\n\n    return translit_string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def separate_to_lists(dataframe):\n    comment_normal = []\n    comment_toxic = []\n    size = dataframe.shape[0]\n\n    for i in range(size):\n        comment = dataframe['comment'][i]\n        lang = dataframe['lang'][i]\n        comment = clean_text(comment)\n        if lang != 'en':\n            comment = translit(comment)            \n        if dataframe['toxic'][i] == 0:\n            comment_normal.append(comment)\n        else:\n            comment_toxic.append(comment)\n        \n    normar_labels = np.zeros(len(comment_normal))\n    toxic_labels = np.ones(len(comment_toxic))\n    \n    return comment_normal, comment_toxic, normar_labels, toxic_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comment_normal, comment_toxic, labels_normal, labels_toxic = separate_to_lists(mini_multilang_trainset_combo)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# again preprocess text data\n\ndef preprocess(X_batch, y_batch):\n    n_words = 128\n    shape = tf.shape(X_batch) * tf.constant([1, 0]) + tf.constant([0, n_words])\n    X_batch = tf.strings.substr(X_batch, 0, 300)\n    X_batch = tf.strings.lower(X_batch)\n    X_batch = tf.strings.regex_replace(X_batch, rb\"<br\\s*/?>\", b\" \")\n    X_batch = tf.strings.regex_replace(X_batch, b\"[^a-zA-Z']\", b\" \")\n    X_batch = tf.strings.split(X_batch)\n    X_batch =X_batch.to_tensor(shape=shape, default_value=b\"<pad>\")\n    return X_batch, y_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make tensorflow datasets\n\ndataset_normal =  tf.data.Dataset.from_tensor_slices((tf.constant(comment_normal, dtype=tf.string), \n                                                      tf.constant(labels_normal, dtype=tf.float32)))\ndataset_toxic =  tf.data.Dataset.from_tensor_slices((tf.constant(comment_toxic, dtype=tf.string), \n                                                     tf.constant(labels_toxic, dtype=tf.float32)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\ntoxic_vocabulary = Counter()\nfor X_batch, y_batch in dataset_toxic.batch(32).map(preprocess):\n    for comment in X_batch:\n        toxic_vocabulary.update(list(comment.numpy()))\n        \nnormal_vocabulary = Counter()\nfor X_batch, y_batch in dataset_normal.batch(32).map(preprocess):\n    for comment in X_batch:\n        normal_vocabulary.update(list(comment.numpy()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_vocabulary.most_common()[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normal_vocabulary.most_common()[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(toxic_vocabulary), len(normal_vocabulary)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will combine toxic and normal vacabularies","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"new_toxic_vocabulary = toxic_vocabulary\nnew_normal_vocabulary = normal_vocabulary\n\ntoxic_vocabulary_list = list(new_toxic_vocabulary)\nnormal_vocabulary_list = list(new_normal_vocabulary)\n\nfor word in normal_vocabulary_list:\n    if new_toxic_vocabulary[word] != 0:\n        del new_toxic_vocabulary[word]\n\nlen(new_toxic_vocabulary), len(new_normal_vocabulary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_toxic_vocabulary.most_common()[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normal_vocab_size = 40000\ntruncated_normal_vocabulary = [\n    word for word, count in new_normal_vocabulary.most_common()[:normal_vocab_size]]\n\ntoxic_vocab_size = 20000\ntruncated_toxic_vocabulary = [\n    word for word, count in new_toxic_vocabulary.most_common()[:toxic_vocab_size]]\n\nmerged_vocabulary = truncated_normal_vocabulary + truncated_toxic_vocabulary\nlen(merged_vocabulary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_to_id = {word: index for index, word in enumerate(merged_vocabulary)}\n\nvocab_size = normal_vocab_size + toxic_vocab_size\n\nfor word in b\"fuck this shit i hate it\".split():\n    print(word_to_id.get(word) or vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for word in b\"pochel v pizda urod\".split():\n    print(word_to_id.get(word) or vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_to_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_oov_buckets = 5000\n\nwords = tf.constant(merged_vocabulary)\nword_ids = tf.range(len(merged_vocabulary), dtype=tf.int64)\nvocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n\ntable = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)\n\ntable.lookup(tf.constant([b\"fuck this shit i hate it\".split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare efficient tensorflow train set\n\nall_comments = comment_normal + comment_toxic\nall_labels = np.concatenate((np.zeros(len(comment_normal)), np.ones(len(comment_toxic))), axis = 0)\n\ntrain_dataset=  tf.data.Dataset.from_tensor_slices((tf.constant(all_comments, dtype=tf.string), \n                                                      tf.constant(all_labels, dtype=tf.float32)))\n\ndef encode_words(X_batch, y_batch):\n    return table.lookup(X_batch), y_batch\n\nbatch_size = 128\n\ntrain_set = train_dataset.repeat().shuffle(50000).batch(batch_size).map(preprocess)\ntrain_set = train_set.map(encode_words).prefetch(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will train our model with simple GRU cells on this mini train set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(42)\ntf.random.set_seed(42)\n\ntrain_size = len(all_comments)\nembed_size = 128\nmodel = keras.models.Sequential([\n    keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n                           mask_zero=True,\n                           input_shape=[None]),\n    keras.layers.GRU(128, return_sequences=True),\n    keras.layers.GRU(128),\n    keras.layers.Dense(1, activation=\"sigmoid\")\n])\n\noptimizer=tf.keras.optimizers.Nadam(learning_rate=0.001)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n\nhistory = model.fit(train_set, steps_per_epoch=train_size // batch_size, epochs=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(output_dir + \"model.h5\")\n#model.load_weights(output_dir + \"model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.arange(len(history.history[\"loss\"])) + 0.5, history.history[\"loss\"], \"b.-\", label=\"Training loss\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.arange(len(history.history[\"accuracy\"])) + 0.5, history.history[\"accuracy\"], \"b.-\", label=\"Training accuracy\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_list = []\nlabel_list = []\n\nnum_taken = all_labels.shape[0]//batch_size\n\nfor (X_batch, y_batch) in train_set.take(num_taken):\n    batch_predictions = model.predict(X_batch)\n    \n    for prediction in batch_predictions:\n        pred_list.append(prediction)\n        \n    for label in y_batch:\n        label_list.append(label)\n        \ny_pred = np.asarray(pred_list).reshape((len(pred_list),))\ny_train = np.asarray(label_list).reshape((len(label_list),))\ny_train_pred = np.around(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ncm = confusion_matrix(y_train, y_train_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score, recall_score\n\nprint(\"accuracy_score: \" + str(accuracy_score(y_train, y_train_pred)))\nprint(\"precision_score: \" + str(precision_score(y_train, y_train_pred)))\nprint(\"recall_score: \" + str(recall_score(y_train, y_train_pred)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now we will prepare validation set and perform evaluation on it.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"VALIDATION DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare validation set\n\nvalid_data_b = load_csv_data(input_dir,\"validation-processed-seqlen128\")\n\nvalid_data_b['lang'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def separate_to_lists_2(dataframe):\n    comments = []\n    labels = []\n    size = dataframe.shape[0]\n\n    for i in range(size):\n        lang = dataframe['lang'][i]\n        label = dataframe['toxic'][i]\n        comment = dataframe['comment_text'][i]\n        comment = clean_text(comment)\n        if lang != 'en':\n            comment = translit(comment)                \n        comments.append(comment)\n        labels.append(label)\n    \n    return comments, labels\n\nvalid_comments, valid_labels = separate_to_lists_2(valid_data_b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tensorflow dataset\n\ndataset_valid =  tf.data.Dataset.from_tensor_slices((tf.constant(valid_comments, dtype=tf.string), \n                                                      tf.constant(valid_labels, dtype=tf.float32)))\n\nvalid_set = dataset_valid.repeat().batch(batch_size).map(preprocess)\nvalid_set = valid_set.map(encode_words).prefetch(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_taken = len(valid_labels)//batch_size\n\nmodel.evaluate(valid_set.take(num_taken))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_pred_list = []\nvalid_label_list = []\n\nnum_taken = len(valid_labels)//batch_size\n\nfor (X_batch, y_batch) in valid_set.take(num_taken):\n    batch_predictions = model.predict(X_batch)\n    \n    for prediction in batch_predictions:\n        valid_pred_list.append(prediction)\n        \n    for label in y_batch:\n        valid_label_list.append(label)\n        \ny_valid_prob = np.asarray(valid_pred_list).reshape((len(valid_pred_list),))\ny_valid = np.asarray(valid_label_list).reshape((len(valid_label_list),))\ny_valid_pred = np.around(y_valid_prob)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(y_valid, y_valid_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"accuracy_score: \" + str(accuracy_score(y_valid, y_valid_pred)))\nprint(\"precision_score: \" + str(precision_score(y_valid, y_valid_pred)))\nprint(\"recall_score: \" + str(recall_score(y_valid, y_valid_pred)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ok. 77% of accuracy is not bed, but precision and recall scores are so low. We will continue to train model on validation set, I reckon this is not \"cheating\"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# continue training on validation data\n\ntrain_size = len(valid_labels)\n\nhistory = model.fit(valid_set, steps_per_epoch=train_size // batch_size, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_pred_list = []\nvalid_label_list = []\n\nnum_taken = len(valid_labels)//batch_size\n\nfor (X_batch, y_batch) in valid_set.take(num_taken):\n    batch_predictions = model.predict(X_batch)\n    \n    for prediction in batch_predictions:\n        valid_pred_list.append(prediction)\n        \n    for label in y_batch:\n        valid_label_list.append(label)\n        \ny_valid_prob = np.asarray(valid_pred_list).reshape((len(valid_pred_list),))\ny_valid = np.asarray(valid_label_list).reshape((len(valid_label_list),))\ny_valid_pred = np.around(y_valid_prob)\n\ncm = confusion_matrix(y_valid, y_valid_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ok, I suppose it was considerably improve situation. We can start working with test dataset.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"TEST DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_b = load_csv_data(input_dir,\"test-processed-seqlen128\")\ntest_data_b.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_b.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def separate_to_lists_test_data(dataframe):\n    comments = []\n    size = dataframe.shape[0]\n\n    for i in range(size):\n        comment = dataframe['comment_text'][i]\n        comment = clean_text(comment)\n        comment = translit(comment)\n        comments.append(comment)\n    \n    return comments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_comments = separate_to_lists_test_data(test_data_b)\npseudo_labels = np.zeros(len(test_comments))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_comments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_test =  tf.data.Dataset.from_tensor_slices((tf.constant(test_comments, dtype=tf.string), \n                                                      tf.constant(pseudo_labels, dtype=tf.float32)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set = dataset_test.batch(batch_size=1).map(preprocess)\ntest_set = test_set.map(encode_words).prefetch(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred_list = []\n#counter = 0\nfor data in test_set.as_numpy_iterator():\n    X, y = data\n    prediction = model.predict(X)\n    test_pred_list.append(prediction)\n    #counter += 0\n    #if counter%5000 == 0:\n        #print(str(counter/63812) +\"%\")\nlen(test_pred_list)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_prob = np.asarray(test_pred_list).reshape((len(test_pred_list),))\n#y_test_pred = np.around(y_test_prob)\ny_test_pred = y_test_prob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counter = 0\n\nfor prediction in y_test_pred:\n    if prediction >= 0.5:\n        counter+=1\n        \ncounter, counter/len(y_test_pred), len(y_test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = load_csv_data(input_dir, \"sample_submission\")\nsubmission['toxic'] = y_test_pred\nsubmission.to_csv(output_dir + 'submission.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}