{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"import torch\n\n# If there's a GPU available...\nif torch.cuda.is_available():    \n\n    # Tell PyTorch to use the GPU.    \n    device = torch.device(\"cuda\")\n\n    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n\n    print('We will use the GPU:', torch.cuda.get_device_name(0))\n    !nvidia-smi\n\n# If not...\nelse:\n    print('No GPU available, using the CPU instead.')\n    device = torch.device(\"cpu\")\n","metadata":{"id":"n1ksa8tUE_Yk","outputId":"69ea8f71-67a8-4bc6-b375-a3bb338eae52"}},{"cell_type":"code","source":"!pip install transformers\n!git clone https://github.com/aub-mind/arabert\n!pip install -r arabert/requirements.txt\n\n!pip install transformers==4.12.2\n!pip install farasapy==0.0.14\n!pip install pyarabic==0.6.14\n!git clone https://github.com/aub-mind/arabert\n!pip install emoji==1.6.1\n!pip install sentencepiece==0.1.96","metadata":{"id":"gABfbzd7ZeeX","outputId":"9568ab92-c718-4095-e03a-70ed1773d4b1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom typing import List\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom transformers import AutoTokenizer, AutoModel\nfrom arabert.preprocess import ArabertPreprocessor\nimport numpy as np\nimport torch\nimport random\nimport matplotlib.pyplot as plt\nimport copy\nfrom arabert.preprocess import ArabertPreprocessor\nfrom sklearn.metrics import (accuracy_score, classification_report,\n                             confusion_matrix, f1_score, precision_score,\n                             recall_score, cohen_kappa_score)\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import (AutoConfig, AutoModelForSequenceClassification,\n                          AutoTokenizer, BertTokenizer, Trainer,\n                          TrainingArguments)\nfrom transformers.data.processors.utils import InputFeatures","metadata":{"id":"_exJv80YmSDN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"aubmindlab/bert-base-arabertv02\"\narabert_prep = ArabertPreprocessor(model_name=model_name)\nmodel = AutoModel.from_pretrained(model_name)\ntokenizer = AutoTokenizer.from_pretrained(model_name)","metadata":{"id":"VDceln104DoS","outputId":"4a8ccfe4-f1aa-427d-fa79-2d102454983b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = pd.read_csv(\"/kaggle/input/ml-olympiad-dialectrecognition/train.csv\") # , delimiter=';'\ndataset","metadata":{"id":"Q5RVMpIiX7lo","outputId":"edb78bca-5a50-4c3f-b818-cd4c1220f752"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = dataset.drop(columns=[\"FileName\"])\ndataset = dataset.drop(columns=[\"SegmentID\"])\ndataset = dataset.drop(columns=[\"SegmentStart\"])\ndataset = dataset.drop(columns=[\"SegmentEnd\"])\ndataset = dataset.drop(columns=[\"Speaker\"])\ndataset = dataset.drop(columns=[\"GroundTruthText\"])\n","metadata":{"id":"pcNY312dzNfl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[\"SpeakerAge\"].value_counts()","metadata":{"id":"Vkn0ng1e2B3n","outputId":"d9a33959-70ca-4d47-9eb9-3b891dd04bb5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[\"SpeakerAge\"].value_counts()","metadata":{"id":"l3_Nv6FRF9yu","outputId":"4497146a-1a32-483d-803d-a83c2aed2a47"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleanup_Dialect = {\"SpeakerDialect\":     {\"Najdi\": 1, \"Hijazi\": 2, \"Khaliji\": 3, \"ModernStandardArabic\": 4},\n                   \"SpeakerGender\": {\"Male\":\"ذكر\", \"Female\": \"أنثى\", \"Unknown\": \"غير معروف\"}}\ndataset = dataset.replace(cleanup_Dialect)\ndataset.head()","metadata":{"id":"HO2dfEdcyVZN","outputId":"487d4403-4dbe-45c8-a42b-cf41c37fd508"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(dataset[\"ShowName\"])):\n  s = dataset[\"ShowName\"][i]\n  s = s.split(\"-\")\n  if len(s) == 1:\n    dataset[\"ShowName\"][i] = s\n  else:\n    dataset[\"ShowName\"][i] = s[0].strip()\n\n\n  s = dataset[\"SpeakerAge\"][i]\n  s = s.split(\"--\")\n  if len(s) == 1:\n    dataset[\"SpeakerAge\"][i] = \"غير معروف\"\n  else:\n    dataset[\"SpeakerAge\"][i] = s[1].strip()\n\n\n  s = dataset[\"Environment\"][i]\n  s = s.split(\"--\")\n  dataset[\"Environment\"][i] = s[1].strip()\n","metadata":{"id":"cBQ4drGR0oxR","outputId":"91da432e-b528-467e-95e8-9d9c5203f38c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.head()","metadata":{"id":"72YmlFQn1Ou9","outputId":"7db9df15-fdcb-48d6-f4f0-8dc5ac9076f1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    \ndataset.columns\nimport pandas as pd\n\n\na = ['ShowName', 'FullFileLength', 'SegmentLength', 'SpeakerAge',\n       'SpeakerGender', 'SpeakerDialect', 'Environment', 'ProcessedText']\n\nAll_samles = dataset.values.tolist()\n#to_numpy()\nAll_samles.insert(0, a)\n","metadata":{"id":"xRNZxr0y8inY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"All_samles[0][5]","metadata":{"id":"4ZaznA4rudt8","outputId":"efe116b5-d665-4fe8-d792-e117c0b91863"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp1 = \"\"\n\nUnit_1 = []\n\nlabel_list = []\n\nfor i in range(len(All_samles)):\n  if i ==0:\n    continue\n  for j in range(len(All_samles[0])): \n    if j != 5:  ### not Relation and not Unit_2\n      temp1 = temp1 + str(All_samles[i][j]) + \" [SEP] \"\n\n    elif j == 5: ### it is  Relation\n      label_list.append(All_samles[i][j])\n\n  Unit_1.append(temp1)\n  temp1 = \"\"\n\n\n","metadata":{"id":"kAQKsxLjn0Pb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(Unit_1)\n","metadata":{"id":"6ZlgvLP2n0Sb","outputId":"e2eee599-ecec-4046-e82a-edb1e4b65735"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset[\"Unit_1\"] = Unit_1\ndataset\n\n\t","metadata":{"id":"bJ6nOBDcn0V8","outputId":"21452e4d-dc1c-42eb-faa4-a7ab593e154a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arabic_prep = ArabertPreprocessor(model_name)\n\ndataset[\"Unit_1\"] = dataset[\"Unit_1\"].apply(lambda x: arabic_prep.preprocess(x))\n\n","metadata":{"id":"VbRFBmb0F4jw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset","metadata":{"id":"iGP6VTzeGkf6","outputId":"9f76da45-8253-4a5a-911c-2ad4e53284f4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max = 0\ns= \"\"\nfor i in range(len(dataset[\"ProcessedText\"])):\n  #print(dataset[\"ProcessedText\"][i])\n  if dataset[\"ProcessedText\"][i] == \"nan\":\n    continue\n\n  \n  s = str(dataset[\"ProcessedText\"][i]).split()\n  \n\n  if len(s) >= max:\n    max = len(s)\n\nmax","metadata":{"id":"Kd3D-1rEFPfB","outputId":"d99f1a76-2f3b-4994-c68f-5114bb8165e3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This will hold all the downloaded and structred datasets\nall_datasets= []\nDATA_COLUMN = \"Unit_1\" #\"text\"\nLABEL_COLUMN = \"SpeakerDialect\" #\"label\"","metadata":{"id":"5l3S6koqmezp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset:\n    def __init__(\n        self,\n        name: str,\n        train: List[pd.DataFrame],\n        test: List[pd.DataFrame],\n        label_list: List[str],\n    ):\n        self.name = name\n        self.train = train\n        self.test = test\n        self.label_list = label_list","metadata":{"id":"5oC1nkNdnPlA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_label = dataset[[\"Unit_1\",\"SpeakerDialect\"]]  \ndata_label.columns = [DATA_COLUMN, LABEL_COLUMN]\n\ntrain, test = train_test_split(data_label, test_size=0.05)\nlabel_list = data_label['SpeakerDialect'].unique()\n\ndata = CustomDataset(\"Dataset\", train, test, label_list)","metadata":{"id":"eDsZLSFime2n"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok = AutoTokenizer.from_pretrained(model_name)","metadata":{"id":"boJ9BM4EnukI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = max","metadata":{"id":"ZwJdWZVaonLs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClassificationDataset(Dataset):\n    def __init__(self, text, target, model_name, max_len, label_map):\n      super(ClassificationDataset).__init__()\n      self.text = text\n      self.target = target\n      self.tokenizer_name = model_name\n      self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n      self.max_len = max_len\n      self.label_map = label_map\n      \n\n    def __len__(self):\n      return len(self.text)\n\n    def __getitem__(self,item):\n      text = str(self.text[item])\n      text = text\n      \n      #inputs = tokenizer.encode_plus(text[0],text[1], max_length=self.max_len, padding = \"max_length\", truncation = True,  return_attention_mask = True, return_tensors = \"pt\")\n      \n      inputs = self.tokenizer( #################################################\n          text,\n        \n          max_length=self.max_len,\n          truncation=True,\n          padding='max_length'\n          \n      )  \n          \n      return InputFeatures(**inputs,label=self.label_map[self.target[item]])","metadata":{"id":"P7tUj8m-oU3g"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = { v:index for index, v in enumerate(data.label_list) }\nprint(label_map)\n\ntrain_dataset = ClassificationDataset(\n    data.train[DATA_COLUMN].to_list(),\n    data.train[LABEL_COLUMN].to_list(),\n    model_name,\n    max_len,\n    label_map\n  )\ntest_dataset = ClassificationDataset(\n    data.test[DATA_COLUMN].to_list(),\n    data.test[LABEL_COLUMN].to_list(),\n    model_name,\n    max_len,\n    label_map\n  )","metadata":{"id":"YVy32_WtoU6v","outputId":"564dba12-f398-4247-f301-ecbc6a74d6ac"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_init():\n    return AutoModelForSequenceClassification.from_pretrained(model_name, return_dict=True, num_labels=len(label_map))","metadata":{"id":"Ms-MPx2loU9-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(p): #p should be of type EvalPrediction\n  preds = np.argmax(p.predictions, axis=1)\n  assert len(preds) == len(p.label_ids)\n  print(classification_report(p.label_ids,preds))\n  print(confusion_matrix(p.label_ids,preds))\n  macro_f1 = f1_score(p.label_ids,preds,average='weighted')\n  macro_precision = precision_score(p.label_ids,preds,average='weighted')\n  macro_recall = recall_score(p.label_ids,preds,average='weighted')\n  kappa = cohen_kappa_score(p.label_ids,preds)\n  acc = accuracy_score(p.label_ids,preds)\n  return {       \n      'F1-score' : macro_f1,\n      \n      'precision': macro_precision,\n      'recall': macro_recall,\n      'Kappa' : kappa,\n      'accuracy': acc\n  }","metadata":{"id":"7Kn_FVO_oVBC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n  random.seed(seed)\n  np.random.seed(seed)\n  torch.manual_seed(seed)\n  torch.cuda.manual_seed(seed)\n  torch.cuda.manual_seed_all(seed)\n  torch.backends.cudnn.deterministic=True\n  torch.backends.cudnn.benchmark = False","metadata":{"id":"CXmuJPbeo1YN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_args = TrainingArguments( \n    output_dir= \"./train\",    \n    adam_epsilon = 1e-8,\n    learning_rate = 2e-5,\n    fp16 = False, # enable this when using V100 or T4 GPU\n    per_device_train_batch_size = 16, # up to 64 on 16GB with max len of 128\n    per_device_eval_batch_size = 128,\n    gradient_accumulation_steps = 5, # use this to scale batch size without needing more memory\n    num_train_epochs= 8,\n    warmup_ratio = 0,\n    do_eval = True,\n    evaluation_strategy = 'epoch',\n    save_strategy = 'epoch',\n    load_best_model_at_end = True, # this allows to automatically get the best model at the end based on whatever metric we want\n    metric_for_best_model = 'accuracy',\n    greater_is_better = True,\n    seed = 25\n  )\n\nset_seed(training_args.seed)","metadata":{"id":"Ry8zFI8-o1b3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model = model_init(),\n    args = training_args,\n    train_dataset = train_dataset,\n    eval_dataset=test_dataset,\n    compute_metrics=compute_metrics,\n)","metadata":{"id":"HdIJMJyMo1fQ","outputId":"944caae2-eb6c-44ab-e8d2-db9c1505088a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#start the training\ntrainer.train()","metadata":{"id":"-Co-uGgOo1i0","outputId":"6f40c2e5-e612-4658-bfd0-dce80ab497ad"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"cZ1KPyzJRNgW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"fcNshEOpRNjT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"oZudhY6MRNmw"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing phase","metadata":{"id":"cX701NyvOxNp"}},{"cell_type":"code","source":"testData = pd.read_csv(\"/kaggle/input/ml-olympiad-dialectrecognition/test.csv\") # , delimiter=';'\ntestData.head()","metadata":{"outputId":"32e29e23-bff6-429e-96e2-833e869cfd99","id":"3ka3f8FFp0gk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testData = testData.drop(columns=[\"FileName\"])\nSegmentID = testData[\"SegmentID\"]\ntestData = testData.drop(columns=[\"SegmentID\"])\ntestData = testData.drop(columns=[\"SegmentStart\"])\ntestData = testData.drop(columns=[\"SegmentEnd\"])\ntestData = testData.drop(columns=[\"Speaker\"])\ntestData = testData.drop(columns=[\"GroundTruthText\"])\n\ntestData[\"SegmentID\"] = SegmentID\n","metadata":{"id":"rM9eKwZ9p0gk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testData[\"SpeakerAge\"].value_counts()","metadata":{"outputId":"007528be-fba4-4254-980e-393e05e8e3a2","id":"jxh3Lyc0p0gk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleanup_Dialect = {\n                   \"SpeakerGender\": {\"Male\":\"ذكر\", \"Female\": \"أنثى\", \"Unknown\": \"غير معروف\"}}\ntestData = testData.replace(cleanup_Dialect)\ntestData.head()","metadata":{"outputId":"5c359ecf-de88-4012-da1e-6d7b2688d725","id":"GDw28-cjp0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(testData[\"ShowName\"])):\n  s = testData[\"ShowName\"][i]\n  s = s.split(\"-\")\n  if len(s) == 1:\n    testData[\"ShowName\"][i] = s\n  else:\n    testData[\"ShowName\"][i] = s[0].strip()\n\n\n  s = testData[\"SpeakerAge\"][i]\n  s = s.split(\"--\")\n  if len(s) == 1:\n    testData[\"SpeakerAge\"][i] = \"غير معروف\"\n  else:\n    testData[\"SpeakerAge\"][i] = s[1].strip()\n\n\n  s = testData[\"Environment\"][i]\n  s = s.split(\"--\")\n  testData[\"Environment\"][i] = s[1].strip()\n","metadata":{"outputId":"c2bb76f6-070d-4321-eed7-2b26984d27ea","id":"FULEIqT9p0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testData.head()","metadata":{"outputId":"4bb3e2be-bf3e-4daf-beb1-2471f2b79544","id":"Lkirxg_Lp0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    \ntestData.columns\n\nimport pandas as pd\n\n\na = ['ShowName', 'FullFileLength', 'SegmentLength', 'SpeakerAge',\n       'SpeakerGender', 'Environment', 'ProcessedText']\n       \n#['ShowName', 'FullFileLength', 'SegmentLength', 'SpeakerAge','SpeakerGender', 'SpeakerDialect', 'Environment', 'ProcessedText']\n\nAll_samles = testData.values.tolist()\n#to_numpy()\nAll_samles.insert(0, a)","metadata":{"id":"Olbcm25bp0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"All_samles[0][5]","metadata":{"outputId":"f1563c91-f0f5-492e-c8e5-c5d710613820","id":"apYEP-0ip0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp1 = \"\"\n\nUnit_1 = []\n\nlabel_list = []\n\nfor i in range(len(All_samles)):\n  if i ==0:\n    continue\n  for j in range(len(All_samles[0])): \n    \n    temp1 = temp1 + str(All_samles[i][j]) + \" [SEP] \"\n\n\n  Unit_1.append(temp1)\n  temp1 = \"\"\n\n\n","metadata":{"id":"4-GDpzItp0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(Unit_1)\n","metadata":{"outputId":"123217b0-f44b-4caf-8735-2d6ff26403ac","id":"UyIm5obfp0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntestData[\"Unit_1\"] = Unit_1\ntestData\n\n\t","metadata":{"outputId":"cf2d5b7a-2cf7-48ea-e0dd-b2ed0a305636","id":"qatOIpY6p0gl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arabic_prep = ArabertPreprocessor(model_name)\n\ntestData[\"Unit_1\"] = testData[\"Unit_1\"].apply(lambda x: arabic_prep.preprocess(x))\n\n","metadata":{"id":"VSDQX_BNp0gm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testData","metadata":{"outputId":"8f23a63e-ad84-44ab-8520-98e96e4a92dc","id":"9oqbKYYFp0gm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = list(dataset[\"SpeakerDialect\"].unique())\n\ndef predict(text, tokenizer):\n \n  #print(text)\n  inputs = tokenizer( \n          text,\n          max_length=max_len,\n          padding='max_length',\n          truncation=True,\n          return_tensors='pt'\n      )   \n\n          \n  #return InputFeatures(**inputs,label=self.label_map[self.target[item]]) \n  \n  input_ids = inputs['input_ids'].to(device)   #(input_ids + ([tokenizer.pad_token_id] * padding_length)).to(device)  \n  attention_mask = inputs['attention_mask'].to(device)  \n  \n\n\n  #print(inputs)\n  output = trainer.model(input_ids, attention_mask)\n\n\n\n  print(output)\n  _, prediction = torch.max(output[0], dim=1)\n  return prediction[0]\n\n\ntokenizer = AutoTokenizer.from_pretrained(model_name)\n\nprediction_list = []\ni = 0\nfor element in testData[\"Unit_1\"]:\n    id = testData[\"SegmentID\"][i]\n  \n    pre = predict(element,tokenizer)\n  \n    pre_txt = label_list[pre]\n   \n    if pre_txt == 'Najdi': pre_txt = 1\n    if pre_txt == 'Hijazi': pre_txt = 2\n    if pre_txt == 'Khaliji': pre_txt = 3\n    if pre_txt == 'ModernStandardArabic': pre_txt = 4\n\n    \n    prediction_list.append(pre_txt)\n    \n    i = i + 1\n  \n#prediction_list","metadata":{"id":"Dr3vEUEW-UhN","outputId":"187262e3-f2a6-4159-d790-454977b7ccf2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame({'SegmentID' : SegmentID, 'SpeakerDialect' : prediction_list},\n                       columns = ['SegmentID', 'SpeakerDialect'])\n\nfinal_results.to_csv(\"for_submission.csv\") # for_submission\n\nfinal_results","metadata":{"id":"E7GUoiv-PM_m","outputId":"9b29d423-3046-4817-89b4-2c18365b3aa7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results[\"SpeakerDialect\"].unique()","metadata":{"id":"g_RKDUqdPNB_","outputId":"ec89f2ce-33aa-49de-dd1e-1565ec4b67a0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"DitBSDPTKsx8"},"execution_count":null,"outputs":[]}]}