{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re \nimport matplotlib.pyplot as plt\n\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.linear_model import Ridge, RidgeCV\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.pipeline import Pipeline\nfrom scipy.stats import rankdata\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-30T19:54:34.590864Z","iopub.execute_input":"2021-11-30T19:54:34.591990Z","iopub.status.idle":"2021-11-30T19:54:34.609483Z","shell.execute_reply.started":"2021-11-30T19:54:34.591933Z","shell.execute_reply":"2021-11-30T19:54:34.608555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text cleaning ","metadata":{}},{"cell_type":"code","source":"def preprocess_text(text, lower_case=True, clean_text=True):\n    \n    if lower_case:\n        text = text.lower()\n    \n    # Remove website links\n    template = re.compile(r'https?://\\S+|www\\.\\S+') \n    text = template.sub(r'', text)\n    \n    # Remove HTML tags\n    template = re.compile(r'<[^>]*>') \n    text = template.sub(r'', text)\n    \n    # Remove none ascii characters\n    template = re.compile(r'[^\\x00-\\x7E]+') \n    text = template.sub(r'', text)\n    \n    # Replace none printable characters\n    template = re.compile(r'[\\x00-\\x0F]+') \n    text = template.sub(r' ', text)\n    \n    if clean_text:\n        # Replace shortenings \n        text = re.sub(r\"what's\", \"what is \", text)\n        text = re.sub(r\"\\'s\", \" \", text)\n        text = re.sub(r\"\\'ve\", \" have \", text)\n        text = re.sub(r\"don't\", \"do not \", text)\n        text = re.sub(r\"n't\", \"n not \", text)\n        text = re.sub(r\"i'm\", \"i am \", text)\n        text = re.sub(r\"\\'re\", \" are \", text)\n        text = re.sub(r\"\\'d\", \" would \", text)\n        text = re.sub(r\"\\'ll\", \" will \", text)\n        # Remove special characters\n        template = re.compile('[\"#$%&\\'()\\*\\+-/:;<=>@\\[\\]\\\\\\\\^_`{|}~]') \n        text = template.sub(r' ', text)\n        # Replace multiple punctuation \n        text = re.sub('[.!?]{2,}', '.', text)\n        text = re.sub(',+', ',', text) \n        # Remove numbers\n        text = re.sub('\\d+', ' ', text) \n        # Remove extra spaces\n        text = re.sub('\\s+', ' ', text)\n    \n    # Remove spaces at the beginning and at the end of string\n    text = text.strip() \n\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:34.611885Z","iopub.execute_input":"2021-11-30T19:54:34.612401Z","iopub.status.idle":"2021-11-30T19:54:34.626031Z","shell.execute_reply.started":"2021-11-30T19:54:34.612353Z","shell.execute_reply":"2021-11-30T19:54:34.625285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"valid_df = pd.read_csv(\"/kaggle/input/jigsaw-toxic-severity-rating/validation_data.csv\")\n\ndef validate_model(model, valid_df=valid_df, clean_text=True):\n    # pre-process the comments in valid_df\n    for col_name in [\"less_toxic\", \"more_toxic\"]:\n        valid_df[col_name] = valid_df[col_name].map(lambda com : preprocess_text(com, clean_text=clean_text))\n    # predict\n    if isinstance(model,  Pipeline):\n        pred_less = model.predict(valid_df[\"less_toxic\"])\n        pred_more = model.predict(valid_df[\"more_toxic\"])\n    # compare\n    return np.mean(pred_less < pred_more)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:34.627188Z","iopub.execute_input":"2021-11-30T19:54:34.627999Z","iopub.status.idle":"2021-11-30T19:54:34.932552Z","shell.execute_reply.started":"2021-11-30T19:54:34.627961Z","shell.execute_reply":"2021-11-30T19:54:34.931915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load first Kaggle competition dataset","metadata":{}},{"cell_type":"code","source":"# load train data \ntrain1_df = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/train.csv\")\nprint(\"Number of training exaples:\", train1_df.shape[0])\n\n# load test data\ntest_lbl_df = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/test_labels.csv\")\ntest_df = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/test.csv\")\ntest_df = test_df.merge(test_lbl_df)\nprint(\"Number of testing exaples:\", test_df.shape[0])\n\n# concat datasets and drop rows without toxicity rating \ntrain1_df = pd.concat([test_df, train1_df], ignore_index=True)\ntrain1_df.drop(columns=[\"id\"], inplace=True)\ntrain1_df = train1_df[train1_df.toxic >= 0]\ntrain1_df.reset_index(inplace=True,drop=True) \nprint(\"Final number of training exaples:\", train1_df.shape[0])\n\ntrain1_df.describe().iloc[:3,]","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:34.933567Z","iopub.execute_input":"2021-11-30T19:54:34.934350Z","iopub.status.idle":"2021-11-30T19:54:39.540875Z","shell.execute_reply.started":"2021-11-30T19:54:34.934310Z","shell.execute_reply":"2021-11-30T19:54:39.539808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make target value ","metadata":{}},{"cell_type":"code","source":"# Lets look at the distribution of toxic and sever-toxic comments\ntoxic_count = train1_df.groupby([\"toxic\", \"severe_toxic\"]).count()\ntoxic_count = toxic_count.assign(prec = np.round(toxic_count.comment_text / sum(toxic_count.comment_text)*100,2))\ntoxic_count = toxic_count[[\"comment_text\", \"prec\"]]\ntoxic_count.columns = [\"count\", \"precentage\"]\ntoxic_count","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:39.543441Z","iopub.execute_input":"2021-11-30T19:54:39.544354Z","iopub.status.idle":"2021-11-30T19:54:39.652147Z","shell.execute_reply.started":"2021-11-30T19:54:39.544316Z","shell.execute_reply":"2021-11-30T19:54:39.651227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at the distribution of the other toxicity indicators \ntrain1_df['toxic_ind'] = train1_df.obscene + train1_df.threat + train1_df.insult + train1_df.identity_hate\ntoxic_count = train1_df.groupby([\"toxic\", \"severe_toxic\", \"toxic_ind\"]).count()\ntoxic_count = toxic_count.assign(prec = np.round(toxic_count.comment_text / sum(toxic_count.comment_text)*100,2))\ntoxic_count = toxic_count[[\"comment_text\", \"prec\"]]\ntoxic_count.columns = [\"count\", \"precentage\"]\ntoxic_count","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:39.653463Z","iopub.execute_input":"2021-11-30T19:54:39.653731Z","iopub.status.idle":"2021-11-30T19:54:39.780862Z","shell.execute_reply.started":"2021-11-30T19:54:39.653699Z","shell.execute_reply":"2021-11-30T19:54:39.779637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the final toxicity indicator\ntrain1_df['toxic_ind'] = (train1_df.obscene + train1_df.threat + \n                          train1_df.insult + train1_df.identity_hate\n                          ).map(lambda x: min(x,3))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:39.782551Z","iopub.execute_input":"2021-11-30T19:54:39.782805Z","iopub.status.idle":"2021-11-30T19:54:39.956800Z","shell.execute_reply.started":"2021-11-30T19:54:39.782776Z","shell.execute_reply":"2021-11-30T19:54:39.955734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set weights for \"toxic\", \"severe_toxic\", \"toxic_ind\"\nw_toxic = [1, 1, 0.25]\n\n# Define the toxicity score\ntrain1_df = train1_df.assign(y =  w_toxic[0] * train1_df.toxic + \n                             w_toxic[1] * train1_df.severe_toxic +\n                             w_toxic[2] * train1_df.toxic_ind )\ntrain1_df[\"y\"] = train1_df[\"y\"] / max(train1_df['y'])\n\ntrain1_df.loc[train1_df[\"y\"]>0, \"y\"].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:39.958004Z","iopub.execute_input":"2021-11-30T19:54:39.958226Z","iopub.status.idle":"2021-11-30T19:54:40.578218Z","shell.execute_reply.started":"2021-11-30T19:54:39.958200Z","shell.execute_reply":"2021-11-30T19:54:40.577546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define tf-idf + regression model","metadata":{}},{"cell_type":"code","source":"def run_tfidf_model(train_df, max_features=5000, alphas=[0.5, 1, 2]):\n\n    # clean the comment_text in train_df \n    train_df['comment_text'] = train_df['comment_text'].map(lambda x: preprocess_text(x))\n\n    # make model\n    print('Vectorization\\n')\n    vectorizer = TfidfVectorizer(max_features=max_features, stop_words='english') #, min_df= 1e-3,  max_df=0., analyzer = 'char_wb', ngram_range = (3,5)\n    regr = RidgeCV(alphas=alphas)\n    model = Pipeline(\n        [\n            (\"vectorizer\", vectorizer),\n            (\"regr\", regr),\n        ])\n    \n    # Fit the model\n    print('Fit Model\\n')\n    model.fit(train_df['comment_text'], train_df['y'])\n    \n    # Validate the model\n    print('Validate Model')\n    right_order_pred = validate_model(model)\n    print('Correctly ordered sentences in the validation data:', np.round(right_order_pred*100, 3), '%\\n' )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:40.579378Z","iopub.execute_input":"2021-11-30T19:54:40.580059Z","iopub.status.idle":"2021-11-30T19:54:40.589828Z","shell.execute_reply.started":"2021-11-30T19:54:40.579977Z","shell.execute_reply":"2021-11-30T19:54:40.588812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time tfidf_model = run_tfidf_model(train1_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:54:40.590950Z","iopub.execute_input":"2021-11-30T19:54:40.591345Z","iopub.status.idle":"2021-11-30T19:55:59.601365Z","shell.execute_reply.started":"2021-11-30T19:54:40.591316Z","shell.execute_reply":"2021-11-30T19:55:59.600452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf_model[\"regr\"].alpha_","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:55:59.606844Z","iopub.execute_input":"2021-11-30T19:55:59.609436Z","iopub.status.idle":"2021-11-30T19:55:59.657265Z","shell.execute_reply.started":"2021-11-30T19:55:59.609380Z","shell.execute_reply":"2021-11-30T19:55:59.655418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make submission","metadata":{}},{"cell_type":"code","source":"comments_to_score = pd.read_csv(\"/kaggle/input/jigsaw-toxic-severity-rating/comments_to_score.csv\")\n\ntest_score = tfidf_model.predict(comments_to_score[\"text\"])\ncomments_to_score[\"score\"] = rankdata(test_score, method=\"ordinal\")\ncomments_to_score[[\"comment_id\",\"score\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T19:55:59.658497Z","iopub.status.idle":"2021-11-30T19:55:59.658931Z","shell.execute_reply.started":"2021-11-30T19:55:59.658691Z","shell.execute_reply":"2021-11-30T19:55:59.658710Z"},"trusted":true},"execution_count":null,"outputs":[]}]}