{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# !pip install transformers==2.1\nimport transformers\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\nimport datetime\nfrom scipy import sparse\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom string import punctuation\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom tqdm import tqdm\nimport datasets\n%matplotlib inline\nfrom transformers import TFBertModel,TFDistilBertModel\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import OneHotEncoder,LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import log_loss\nfrom transformers import AutoModel, AutoTokenizer\nimport torch\nfrom sklearn import svm\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import BernoulliNB,MultinomialNB\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score, precision_score,recall_score\nfrom transformers import AutoTokenizer\nimport tensorflow as tf\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-20T05:26:48.117391Z","iopub.execute_input":"2022-07-20T05:26:48.118016Z","iopub.status.idle":"2022-07-20T05:26:48.135816Z","shell.execute_reply.started":"2022-07-20T05:26:48.117983Z","shell.execute_reply":"2022-07-20T05:26:48.134838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checkpoint = 'bert-base-cased'\n# checkpoint = 'distilbert-base-cased'\n# checkpoint = '../input/bertcasedbased/bert-base-cased'\n# model = AutoModel.from_pretrained(checkpoint)\n# model.save_pretrained('../input/bertcasedbased/hugging_face_bert-base-cased/')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:00:53.813426Z","iopub.execute_input":"2022-07-19T17:00:53.814145Z","iopub.status.idle":"2022-07-19T17:00:58.833277Z","shell.execute_reply.started":"2022-07-19T17:00:53.814107Z","shell.execute_reply":"2022-07-19T17:00:58.832095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cat /kaggle/input/feedback-prize-effectiveness/train.csv\n# train_data = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\n# train = train_data.copy()\n# train = datasets.Dataset.from_dict(train)\n# train_validation = train.train_test_split(train_size = 0.8)\n# train_validation['validation'] = train_validation.pop('test')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:12:36.131454Z","iopub.execute_input":"2022-07-20T05:12:36.131826Z","iopub.status.idle":"2022-07-20T05:12:36.442670Z","shell.execute_reply.started":"2022-07-20T05:12:36.131796Z","shell.execute_reply":"2022-07-20T05:12:36.441307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Exploration**","metadata":{}},{"cell_type":"code","source":"# train_validation.set_format('pandas')\n# df = train_validation['train'][::]\n# df_summary = df.pivot_table(index='discourse_type',\n#                    columns='discourse_effectiveness',\n#                    values='discourse_id',\n#                    aggfunc= len,margins=True)\n# df_summary.reset_index(inplace=True)\n# df_summary['Adequate'] = df_summary['Adequate']/df_summary.All\n# df_summary['Effective'] = df_summary['Adequate']/df_summary.All\n# df_summary['Ineffective'] = df_summary['Ineffective']/df_summary.All\n# df_summary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data prep all features**","metadata":{}},{"cell_type":"code","source":"# possible_labels = train_data['discourse_effectiveness'].unique()\n# label_dict = {}\n# for index, possible_label in enumerate(possible_labels):\n#     label_dict[possible_label] = index\n# label_dict\n# train_data['discourse_effectiveness'] = train_data.discourse_effectiveness.replace(label_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def train_data_prep(df):\n#     combind_feat = []\n#     label = []\n#     for index,row in df.iterrows():\n#         combined = ''\n#         combined += row['discourse_text']\n#         combined += 'discourse_type is {}'.format(row['discourse_type'])\n#         combind_feat.append(combined)\n#         label.append(row['discourse_effectiveness'])\n#         data = pd.DataFrame({'feature':combind_feat,'label':label})\n#         data['feature'] = data['feature'].str.replace('\\n','')\n#         data['feature'] = data['feature'].str.replace('\\xa0','')\n#         data['feature'].str.lower()\n#     return \n# train_data = train_data_prep(train_data)\n# valid_data = train_data_prep(train_validation['validation'][::])\n# valid_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:14:03.994277Z","iopub.execute_input":"2022-07-20T05:14:03.994650Z","iopub.status.idle":"2022-07-20T05:14:20.246014Z","shell.execute_reply.started":"2022-07-20T05:14:03.994617Z","shell.execute_reply":"2022-07-20T05:14:20.244719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = datasets.Dataset.from_dict(train_data)\n# train_validation = train_data.train_test_split(train_size = 0.8)\n# train_validation['validation'] = train_validation.pop('test')\n# train_validation","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:01:25.578864Z","iopub.execute_input":"2022-07-19T17:01:25.579308Z","iopub.status.idle":"2022-07-19T17:01:25.699046Z","shell.execute_reply.started":"2022-07-19T17:01:25.579274Z","shell.execute_reply":"2022-07-19T17:01:25.698176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tokenizer**","metadata":{}},{"cell_type":"code","source":"# tokenize = AutoTokenizer.from_pretrained(checkpoint)\n# def tokenize_batch(batch):\n#     return tokenize(batch['feature'],padding=True,truncation=True,max_length=256)\n# train_validation_encoded = train_validation.map(tokenize_batch,batch_size=None,batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:06:08.961413Z","iopub.execute_input":"2022-07-19T17:06:08.962886Z","iopub.status.idle":"2022-07-19T17:06:20.774096Z","shell.execute_reply.started":"2022-07-19T17:06:08.962835Z","shell.execute_reply":"2022-07-19T17:06:20.772916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import transformers\n# from transformers import AutoModelForSequenceClassification\n# print([x for x in dir(transformers) if re.search(r'AutoModel',x)])\n\n# device = torch.device('cuda' if torch.cuda.is_available else 'cpu')\n# device = torch.device('cpu')\n# model = (AutoModelForSequenceClassification.from_pretrained(checkpoint,num_labels=3))","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:06:28.225961Z","iopub.execute_input":"2022-07-19T17:06:28.227137Z","iopub.status.idle":"2022-07-19T17:06:29.869721Z","shell.execute_reply.started":"2022-07-19T17:06:28.227080Z","shell.execute_reply":"2022-07-19T17:06:29.868317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# print(TrainingArguments.__doc__)\n# from transformers import DataCollatorForSeq2Seq\n\n# data_collator = DataCollatorForSeq2Seq(tokenize, model=model)\n# train_validation_encoded = train_validation_encoded.remove_columns(\n#     'feature'\n# )\n# features = [train_validation_encoded[\"train\"][i] for i in range(2)]\n# data_collator(features)\n# train_validation_encoded.remove_columns_(train_validation_encoded[\"train\"].column_names)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:06:51.998166Z","iopub.execute_input":"2022-07-19T17:06:51.998560Z","iopub.status.idle":"2022-07-19T17:06:52.009452Z","shell.execute_reply.started":"2022-07-19T17:06:51.998528Z","shell.execute_reply":"2022-07-19T17:06:52.008490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from datasets import DatasetDict\n# tiny_data = DatasetDict()\n# tiny_data['train'] = train_validation_encoded['train'].shuffle(seed=1).select(range(5000))\n# tiny_data['validation'] = train_validation_encoded['validation'].shuffle(seed=1).select(range(500))\n# tiny_data.map(tokenize,batch_size=None,batched=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# out = '../output/bertcasedbased/bert-base-cased'\n# from transformers import Trainer, TrainingArguments\n# batch_size = 8\n# model_name = f'{out}-fine_tuned'\n# logging_steps = len(train_validation_encoded['train'])//batch_size\n# training_args = TrainingArguments(output_dir=model_name,metric_for_best_model='sparse_categorical_crossentropy',\n#                                   num_train_epochs=2,\n#                                   learning_rate=2e-5,\n#                                   per_device_train_batch_size=batch_size,\n#                                   per_device_eval_batch_size=batch_size,\n#                                   weight_decay=0.01,\n#                                   evaluation_strategy=\"epoch\",\n#                                   disable_tqdm=False,\n#                                   logging_steps=100,\n#                                   log_level=\"error\",\n#                                   optim='adamw_torch'\n#                                   )\n# trainer = Trainer(model=model,train_dataset=train_validation_encoded['train'],eval_dataset=train_validation_encoded['validation'], args = training_args, tokenizer=tokenize)\n# torch.cuda.empty_cache()\n# trainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T17:07:14.230565Z","iopub.execute_input":"2022-07-19T17:07:14.231111Z","iopub.status.idle":"2022-07-19T17:07:34.456045Z","shell.execute_reply.started":"2022-07-19T17:07:14.231071Z","shell.execute_reply":"2022-07-19T17:07:34.454215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainer.evaluate()\n# trainer.save_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import normalize\n# prediction = trainer.predict(test_encoded)\n# def NormalizeData(data):\n#     return (data - np.min(data)) / (np.max(data) - np.min(data))\n# prediction_1 = NormalizeData(prediction[0])\n# prediction = normalize(prediction_1,axis=1,norm='l1')\n# prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:27:01.685318Z","iopub.execute_input":"2022-07-20T05:27:01.685761Z","iopub.status.idle":"2022-07-20T05:27:02.021005Z","shell.execute_reply.started":"2022-07-20T05:27:01.685723Z","shell.execute_reply":"2022-07-20T05:27:02.020129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\n# !kaggle kernels output imvision12/tensorflow-feedback-bert-baseline -p /path/to/dest\nAUTO = tf.data.experimental.AUTOTUNE\n# Configuration\nEPOCHS = 3\nBATCH_SIZE = 16\nMAX_LEN = 256\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('../input/bertbaseuncased-1/distil_bert_uncased')\n# Save the loaded tokenizer locally\n# tokenizer.save_pretrained('.')\ndef bert_encode(texts, tokenizer, max_len=MAX_LEN):\n    input_ids = []\n    token_type_ids = []\n    attention_mask = []\n    \n    for text in texts:\n        token = tokenizer(text, max_length=max_len, truncation=True, padding='max_length',\n                         add_special_tokens=True)\n        input_ids.append(token['input_ids'])\n#         token_type_ids.append(token['token_type_ids'])\n        attention_mask.append(token['attention_mask'])\n    return np.array(input_ids), np.array(attention_mask)\n#     return np.array(input_ids), np.array(token_type_ids), np.array(attention_mask)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:27:05.771426Z","iopub.execute_input":"2022-07-20T05:27:05.771868Z","iopub.status.idle":"2022-07-20T05:27:05.904623Z","shell.execute_reply.started":"2022-07-20T05:27:05.771828Z","shell.execute_reply":"2022-07-20T05:27:05.903594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sep = tokenizer.sep_token\ntrain_data['inputs'] = (train_data.discourse_type + sep +train_data.discourse_text).str.lower()\nlabel_dict = {\"discourse_effectiveness\": {\"Ineffective\": 0, \"Adequate\": 1, \"Effective\": 2}}\ntrain_data = train_data.replace(label_dict).rename(columns={'discourse_effectiveness':'label'})\nX_train, X_valid, y_train, y_valid = train_test_split(train_data['inputs'], train_data['label'], test_size=0.12, random_state=42)\nX_train = bert_encode(X_train.astype(str), tokenizer)\nX_valid = bert_encode(X_valid.astype(str), tokenizer)\n\ny_train = y_train.values\ny_valid = y_valid.values","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:27:08.294502Z","iopub.execute_input":"2022-07-20T05:27:08.295004Z","iopub.status.idle":"2022-07-20T05:28:17.794545Z","shell.execute_reply.started":"2022-07-20T05:27:08.294958Z","shell.execute_reply":"2022-07-20T05:28:17.793568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((X_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((X_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:28:49.338360Z","iopub.execute_input":"2022-07-20T05:28:49.338767Z","iopub.status.idle":"2022-07-20T05:28:55.728765Z","shell.execute_reply.started":"2022-07-20T05:28:49.338714Z","shell.execute_reply":"2022-07-20T05:28:55.727892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(bert_model, max_len=MAX_LEN):    \n    input_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n#     token_type_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"token_type_ids\")\n    attention_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n\n    sequence_output = bert_model(input_ids, attention_mask=attention_mask)[0]\n    clf_output = sequence_output[:, 0, :]\n    clf_output = Dropout(.1)(clf_output)\n    out = Dense(3, activation='softmax')(clf_output)\n    \n    model = Model(inputs=[input_ids, attention_mask], outputs=out)\n    model.compile(Adam(lr=1e-4), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n    \n    return model\ntransformer_layer = (TFDistilBertModel.from_pretrained('../input/bertbaseuncased-1/distil_bert_uncased'))\nmodel = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=200,\n    validation_data=valid_dataset,\n    epochs=5\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:36:26.558608Z","iopub.execute_input":"2022-07-20T05:36:26.558985Z","iopub.status.idle":"2022-07-20T05:42:53.245724Z","shell.execute_reply.started":"2022-07-20T05:36:26.558954Z","shell.execute_reply":"2022-07-20T05:42:53.244709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\ntest['inputs'] = (test['discourse_type'] + sep + test['discourse_text']).str.lower()\ntest = bert_encode(test.inputs.astype(str), tokenizer)\nprediction = model.predict(test,verbose=1)\n\nsub = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\nsub['Ineffective'] = prediction[:,0]\nsub['Adequate'] = prediction[:,1]\nsub['Effective'] = prediction[:,2]\nsub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:43:58.991469Z","iopub.execute_input":"2022-07-20T05:43:58.992465Z","iopub.status.idle":"2022-07-20T05:44:01.283453Z","shell.execute_reply.started":"2022-07-20T05:43:58.992422Z","shell.execute_reply":"2022-07-20T05:44:01.282294Z"},"trusted":true},"execution_count":null,"outputs":[]}]}