{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport copy\nimport time\nimport random\nimport string\n\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import rankdata","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:47:29.311214Z","iopub.execute_input":"2021-11-22T13:47:29.312162Z","iopub.status.idle":"2021-11-22T13:47:30.092681Z","shell.execute_reply.started":"2021-11-22T13:47:29.312046Z","shell.execute_reply":"2021-11-22T13:47:30.091969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ridge_cv(vex, X, y, X_test, folds, stratified):\n    kf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=123)\n    val_scores = []\n    rmse_scores = []\n    X_less_toxics = []\n    X_more_toxics = []\n\n    preds = []\n\n    for fold, (train_index, val_index) in enumerate(kf.split(X, stratified)):\n        X_train, y_train = X[train_index], y[train_index]\n        X_val, y_val = X[val_index], y[val_index]\n        model = Ridge()\n        model.fit(X_train, y_train)\n\n        rmse_score = mean_squared_error(model.predict(X_val), y_val, squared=False)\n        rmse_scores.append(rmse_score)\n\n        X_less_toxic = vec.transform(df_val['less_toxic'])\n        X_more_toxic = vec.transform(df_val['more_toxic'])\n\n        p1 = model.predict(X_less_toxic)\n        p2 = model.predict(X_more_toxic)\n\n        X_less_toxics.append(p1)\n        X_more_toxics.append(p2)\n\n        val_acc = (p1 < p2).mean()\n        val_scores.append(val_acc)\n\n        pred = model.predict(X_test)\n        preds.append(pred)\n\n        print(f'FOLD:{fold}, rmse_fold:{rmse_score:.5f}, val_acc:{val_acc:.5f}')\n\n    mean_val_acc = np.mean(val_scores)\n    mean_rmse_score = np.mean(rmse_scores)\n\n    p1 = np.mean(np.vstack(X_less_toxics), axis=0)\n    p2 = np.mean(np.vstack(X_more_toxics), axis=0)\n\n    val_acc = (p1 < p2).mean()\n\n    print(f'00F: val_acc:{val_acc:.5f}, mean val_acc:{mean_val_acc:.5f}, mean rmse_score:{mean_rmse_score:.5f}')\n\n    preds = np.mean(np.vstack(preds), axis=0)\n\n    return p1, p2, preds","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:47:55.520676Z","iopub.execute_input":"2021-11-22T13:47:55.521409Z","iopub.status.idle":"2021-11-22T13:47:55.533508Z","shell.execute_reply.started":"2021-11-22T13:47:55.521375Z","shell.execute_reply":"2021-11-22T13:47:55.532806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = '../input/'","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:47:57.993459Z","iopub.execute_input":"2021-11-22T13:47:57.993781Z","iopub.status.idle":"2021-11-22T13:47:58.000944Z","shell.execute_reply.started":"2021-11-22T13:47:57.993746Z","shell.execute_reply":"2021-11-22T13:47:58.000060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val = pd.read_csv(data_path + \"jigsaw-toxic-severity-rating/validation_data.csv\")\ndf_test = pd.read_csv(data_path + \"jigsaw-toxic-severity-rating/comments_to_score.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:48:05.211691Z","iopub.execute_input":"2021-11-22T13:48:05.212339Z","iopub.status.idle":"2021-11-22T13:48:05.808405Z","shell.execute_reply.started":"2021-11-22T13:48:05.212299Z","shell.execute_reply":"2021-11-22T13:48:05.807671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Toxic Comment Classification Challenge data set","metadata":{}},{"cell_type":"code","source":"jf_train_df = pd.read_csv(data_path + \"jigsaw-toxic-comment-classification-challenge/train.csv\")\njf_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:48:07.358446Z","iopub.execute_input":"2021-11-22T13:48:07.359020Z","iopub.status.idle":"2021-11-22T13:48:09.117917Z","shell.execute_reply.started":"2021-11-22T13:48:07.358982Z","shell.execute_reply":"2021-11-22T13:48:09.117235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jf_train_df['y'] = jf_train_df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].max(axis=1)\njf_train_df = jf_train_df.query('y > 0')\nprint(jf_train_df.shape)\njf_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:48:27.391239Z","iopub.execute_input":"2021-11-22T13:48:27.391964Z","iopub.status.idle":"2021-11-22T13:48:27.429868Z","shell.execute_reply.started":"2021-11-22T13:48:27.391926Z","shell.execute_reply":"2021-11-22T13:48:27.429177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"toxic = 1.0\nsevere_toxic = 2.0\nobscene = 1.0\nthreat = 1.0\ninsult = 1.0\nidentity_hate = 2.0\n\ndef create_train (df):\n    df['y'] = df['y'] + df['severe_toxic']*severe_toxic\n    df['y'] = df['y'] + df['obscene']*obscene\n    df['y'] = df['y'] + df['threat']*threat\n    df['y'] = df['y'] + df['insult']*insult\n    df['y'] = df['y'] + df['identity_hate']*identity_hate\n\n    df = df[['comment_text', 'y', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].rename(columns={'comment_text': 'text'})\n\n    # undersampling -> 0인 값은 이미 제거했으므로 일단은 건너뛰기\n    # min_len = (df['y'] >= 1).sum()\n\n    return df\n\ndf = create_train(jf_train_df)\nprint(jf_train_df['y'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:48:42.445527Z","iopub.execute_input":"2021-11-22T13:48:42.446105Z","iopub.status.idle":"2021-11-22T13:48:42.467976Z","shell.execute_reply.started":"2021-11-22T13:48:42.446059Z","shell.execute_reply":"2021-11-22T13:48:42.467154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\n\nvec = TfidfVectorizer(analyzer='char_wb', max_df=0.5, min_df=3, ngram_range=(4, 6))\nX = vec.fit_transform(df['text'])\ny = df['y'].values\nX_test = vec.transform(df_test['text'])\n\nstratified = np.around(y)\n\njf_p1, jf_p2, jf_preds = ridge_cv(vec, X, y, X_test, FOLDS, stratified)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:48:47.319151Z","iopub.execute_input":"2021-11-22T13:48:47.319876Z","iopub.status.idle":"2021-11-22T13:52:03.675778Z","shell.execute_reply.started":"2021-11-22T13:48:47.319840Z","shell.execute_reply":"2021-11-22T13:52:03.675026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Jigsaw Unintended Bias in Toxicity Classification","metadata":{}},{"cell_type":"code","source":"js_train_df = pd.read_csv(data_path + \"jigsaw-unintended-bias-in-toxicity-classification/train.csv\")\njs_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:52:47.477071Z","iopub.execute_input":"2021-11-22T13:52:47.477357Z","iopub.status.idle":"2021-11-22T13:53:08.536079Z","shell.execute_reply.started":"2021-11-22T13:52:47.477328Z","shell.execute_reply":"2021-11-22T13:53:08.535386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"js_train_df['y'] = js_train_df[['target', 'severe_toxicity', 'obscene', 'threat', 'insult', 'identity_attack']].max(axis=1)\nprint(js_train_df.shape)\njs_train_df = js_train_df.query('y > 0')\nprint(js_train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:53:56.200379Z","iopub.execute_input":"2021-11-22T13:53:56.200965Z","iopub.status.idle":"2021-11-22T13:53:56.738988Z","shell.execute_reply.started":"2021-11-22T13:53:56.200929Z","shell.execute_reply":"2021-11-22T13:53:56.738256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"js_train_df['y'] = js_train_df[['severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].sum(axis=1)\njs_train_df['y'] = js_train_df.apply(lambda row: row['target'] if row['target'] <= 0.5 else row['y'], axis=1)\njs_train_df = js_train_df[['comment_text', 'y']].rename(columns={'comment_text': 'text'})\n\n# down sampling\n\ndf = js_train_df\nprint(df['y'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:54:21.683078Z","iopub.execute_input":"2021-11-22T13:54:21.683330Z","iopub.status.idle":"2021-11-22T13:54:33.216201Z","shell.execute_reply.started":"2021-11-22T13:54:21.683302Z","shell.execute_reply":"2021-11-22T13:54:33.215509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\nvec = TfidfVectorizer(analyzer='char_wb', max_df=0.5, min_df=3, ngram_range=(4, 6))\nX = vec.fit_transform(df['text'])\ny = df['y'].values\nX_test = vec.transform(df_test['text'])\n\nstratified = (np.around(y, decimals=1)*10).astype(int)\njs_p1, js_p2, js_preds = ridge_cv(vec, X, y, X_test, FOLDS, stratified)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:54:37.445016Z","iopub.execute_input":"2021-11-22T13:54:37.445287Z","iopub.status.idle":"2021-11-22T14:13:49.964786Z","shell.execute_reply.started":"2021-11-22T13:54:37.445258Z","shell.execute_reply":"2021-11-22T14:13:49.964044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ruddit data set","metadata":{}},{"cell_type":"code","source":"rd_train_df = pd.read_csv(data_path + \"ruddit-jigsaw-dataset/Dataset/ruddit_with_text.csv\")\nrd_train_df['y'] = rd_train_df['offensiveness_score'].map(lambda x : 0.0 if x <=0 else x)\nprint(rd_train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:20:52.332375Z","iopub.execute_input":"2021-11-22T14:20:52.332995Z","iopub.status.idle":"2021-11-22T14:20:52.407783Z","shell.execute_reply.started":"2021-11-22T14:20:52.332958Z","shell.execute_reply":"2021-11-22T14:20:52.406955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd_train_df = rd_train_df.query('y > 0')\nrd_train_df = rd_train_df[['txt', 'y']].rename(columns={'txt': 'text'})\n\nprint(rd_train_df['y'].value_counts())\nprint(rd_train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:20:53.402496Z","iopub.execute_input":"2021-11-22T14:20:53.402870Z","iopub.status.idle":"2021-11-22T14:20:53.417392Z","shell.execute_reply.started":"2021-11-22T14:20:53.402831Z","shell.execute_reply":"2021-11-22T14:20:53.416664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\ndf = rd_train_df\nvec = TfidfVectorizer(analyzer='char_wb', max_df=0.5, min_df=3, ngram_range=(4, 6))\nX = vec.fit_transform(df['text'])\ny = df['y'].values\nX_test = vec.transform(df_test['text'])\n\nstratified = (np.around(y, decimals=1)*10).astype(int)\nrd_p1, rd_p2, rd_preds = ridge_cv(vec, X, y, X_test, FOLDS, stratified)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:21:03.412054Z","iopub.execute_input":"2021-11-22T14:21:03.412323Z","iopub.status.idle":"2021-11-22T14:23:59.584083Z","shell.execute_reply.started":"2021-11-22T14:21:03.412286Z","shell.execute_reply":"2021-11-22T14:23:59.583357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"# 첫 번째 성능 박살나서 일단 제외\n# jf_max = max(jf_p1.max(), jf_p2.max())\njs_max = max(js_p1.max(), js_p2.max())\nrd_max = max(rd_p1.max(), rd_p2.max())\n\n# p1 = jf_p1/jf_max + js_p1/js_max + rd_p1/rd_max\n# p2 = jf_p2/jf_max + js_p2/js_max + rd_p2/rd_max\n\np1 = js_p1/js_max + rd_p1/rd_max\np2 = js_p2/js_max + rd_p2/rd_max\n\nval_acc = (p1 < p2).mean()\nprint(f'Ensemble: val_acc:{val_acc:.5f}')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:25:22.785798Z","iopub.execute_input":"2021-11-22T14:25:22.786085Z","iopub.status.idle":"2021-11-22T14:25:22.795458Z","shell.execute_reply.started":"2021-11-22T14:25:22.786044Z","shell.execute_reply":"2021-11-22T14:25:22.794180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = js_preds/js_max + rd_preds/rd_max\n\ndf_test['score'] = rankdata(score, method='ordinal')\n\ndf_test[['comment_id', 'score']].to_csv(\"submission.csv\", index=False)\n\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:25:26.088002Z","iopub.execute_input":"2021-11-22T14:25:26.088731Z","iopub.status.idle":"2021-11-22T14:25:26.119936Z","shell.execute_reply.started":"2021-11-22T14:25:26.088694Z","shell.execute_reply":"2021-11-22T14:25:26.119162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}