{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\nfrom bs4 import BeautifulSoup\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nimport gc\nimport re \nfrom scipy import sparse\nimport time\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.options.display.max_colwidth=300\npd.options.display.max_columns = 100\n\nfrom sklearn.linear_model import Ridge\n","metadata":{"papermill":{"duration":2.036616,"end_time":"2021-12-31T12:05:46.476168","exception":false,"start_time":"2021-12-31T12:05:44.439552","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport torch\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Cleans text into a basic form for NLP. Operations include the following:-\n    1. Remove special charecters like &, #, etc\n    2. Removes extra spaces\n    3. Removes embedded URL links\n    4. Removes HTML tags\n    5. Removes emojis\n    \n    text - Text piece to be cleaned.\n    '''\n    template = re.compile(r'https?://\\S+|www\\.\\S+') #Removes website links\n    text = template.sub(r'', text)\n    \n    soup = BeautifulSoup(text, 'lxml') #Removes HTML tags\n    only_text = soup.get_text()\n    text = only_text\n    \n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               u\"\\U00002702-\\U000027B0\"\n                               u\"\\U000024C2-\\U0001F251\"\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    \n    text = re.sub(r\"[^a-zA-Z\\d]\", \" \", text) #Remove special Charecters\n    text = re.sub(' +', ' ', text) #Remove Extra Spaces\n    text = text.strip() # remove spaces at the beginning and at the end of string\n\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # jigsaw1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"toxic_comment_path = '../input/jigsaw-toxic-comment-classification-challenge/'\n\ndf_test = pd.read_csv(toxic_comment_path+'test.csv.zip')\n\ndf_test_label = pd.read_csv(toxic_comment_path+'test_labels.csv.zip').replace(-1,0)\n\ndf_test = pd.merge(df_test, df_test_label, how=\"left\", on = \"id\")\nprint(df_test.shape)\n\ndf_train = pd.read_csv(toxic_comment_path+'train.csv.zip')\n\ndf = pd.concat([df_train, df_test]).rename(columns={'comment_text': 'text'}).reset_index(drop=True)\nprint(df.shape)\n\ndel df_train, df_test, df_test_label; gc.collect()\n\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation data \n\ndf_val = pd.read_csv(\"../input/jigsaw-toxic-severity-rating/validation_data.csv\")\nprint(df_val.shape)\n\ndf_more = df_val[['more_toxic']].rename(columns={'more_toxic': 'text'}).reset_index(drop=True)\ndf_less = df_val[['less_toxic']].rename(columns={'less_toxic': 'text'}).reset_index(drop=True)\n\ndf_val_unique = pd.concat([df_more, df_less]).drop_duplicates(subset='text', keep='first')\n\nprint(df_val_unique.shape)\ndf_val_unique.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicate_index = pd.merge(df, df_val_unique, on='text')['id']\nprint(len(duplicate_index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jig1_no_jig4_dup_df = df[~df['id'].isin(duplicate_index)].reset_index(drop=True)\n\nprint(jig1_no_jig4_dup_df.shape)\njig1_no_jig4_dup_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jig1_no_jig4_dup_df.to_csv('jig1_no_jig4_dup_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jigsaw2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\n    '../input/jigsaw-unintended-bias-in-toxicity-classification/all_data.csv',\n    usecols=['id', 'comment_text',\n       'toxicity', 'severe_toxicity', 'obscene', 'sexual_explicit',\n       'identity_attack', 'insult', 'threat', \n       'male', 'female', 'transgender',\n       'other_gender', 'heterosexual', 'homosexual_gay_or_lesbian', 'bisexual',\n       'other_sexual_orientation', 'christian', 'jewish', 'muslim', 'hindu',\n       'buddhist', 'atheist', 'other_religion', 'black', 'white', 'asian',\n       'latino', 'other_race_or_ethnicity', 'physical_disability',\n       'intellectual_or_learning_disability', 'psychiatric_or_mental_illness',\n       'other_disability']\n).rename(columns={'comment_text': 'text'})\n\nduplicate_index = pd.merge(df, df_val_unique, on='text')['id']\nprint(len(duplicate_index))\n\njig2_no_jig4_dup_df = df[~df['id'].isin(duplicate_index)].reset_index(drop=True)\n\nprint(jig2_no_jig4_dup_df.shape)\njig2_no_jig4_dup_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jig2_no_jig4_dup_df[~jig2_no_jig4_dup_df['homosexual_gay_or_lesbian'].isnull()].tail()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jig2_no_jig4_dup_df.to_csv('jig2_no_jig4_dup_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ruddit","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\n    '../input/ruddit-jigsaw-dataset/Dataset/ruddit_with_text.csv',\n).rename(columns={'comment_id': 'id', 'txt': 'text', 'offensiveness_score': 'y'})\n\nprint(df.shape)\n\nduplicate_index = pd.merge(df, df_val_unique, on='text')['id']\nprint(len(duplicate_index))\n\nruddit_no_jig4_dup_df = df[~df['id'].isin(duplicate_index)].reset_index(drop=True)\n\nprint(ruddit_no_jig4_dup_df.shape)\nruddit_no_jig4_dup_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ruddit_no_jig4_dup_df.to_csv('ruddit_no_jig4_dup_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# good score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\n    '../input/jigsaw-rate-severity-good-score-train-dataset/train_data.csv',\n).rename(columns={'Unnamed: 0.1': 'id'})\n\ndel df['Unnamed: 0']\nprint(df.shape)\n\nduplicate_index = pd.merge(df, df_val_unique, on='text')['id']\nprint(len(duplicate_index))\n\ngood_no_jig4_dup_df = df[~df['id'].isin(duplicate_index)].reset_index(drop=True)\n\nprint(good_no_jig4_dup_df.shape)\ngood_no_jig4_dup_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"good_no_jig4_dup_df.to_csv('good_no_jig4_dup_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}