{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 英語の文章を一旦他言語に翻訳し、その後再び英語に戻すことで、意味を保ったまま少し異なる文章を生成するスクリプトです。Data Augmentationで使えないかと思い実装しました。\n## 精度が上がるかわかりませんが、皆様の参考になれば幸いです。","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.optimizers import Adam\nimport transformers\nimport tqdm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_tr = pd.read_csv('/kaggle/input/feedback-prize-effectiveness/train.csv')\ndataset_te = pd.read_csv('/kaggle/input/feedback-prize-effectiveness/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T00:06:54.409243Z","iopub.execute_input":"2022-07-24T00:06:54.410110Z","iopub.status.idle":"2022-07-24T00:06:54.726115Z","shell.execute_reply.started":"2022-07-24T00:06:54.410060Z","shell.execute_reply":"2022-07-24T00:06:54.724911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#20220627_データ水増し処理-----------------------------\n#---------------------------------------------------\n# augmentation\nfrom joblib import Parallel, delayed\nfrom textblob import TextBlob\nfrom textblob.translate import NotTranslated\nfrom time import sleep \nimport random\n\n\nclass Argumentation_text:\n    \n    def __init__(self, language_list=None): \n        self.language_list=language_list\n    \n    def make_all_label_same_amt(self, df):\n        amount_dic, label_num=self._calc_num_enlarge(df)\n        \n        print(amount_dic)\n        print(label_num)\n        \n        df_output=pd.DataFrame()\n        for label, num_enlarge in amount_dic.items():\n            shou= num_enlarge // label_num[label]\n            amari=num_enlarge % label_num[label]\n            \n            for i in range(shou+1):\n                if i==0:\n                    df_enlarged_dt=self.process_arg(df, {label: amari}, self.language_list[i])\n                else:\n                    df_enlarged_dt=self.process_arg(df, {label: label_num[label]}, self.language_list[i])\n                \n                df_output=pd.concat([df_output, df_enlarged_dt])\n        \n        return df_output\n                    \n        \n    def _calc_num_enlarge(self, df):\n        #各ラベルの数を同じにするために、どのラベルのデータをいくつ水増ししなきゃいけないか計算\n        label_num=df['discourse_effectiveness'].value_counts()\n        max_num_label=label_num.idxmax()\n        other_label=label_num.index.values.tolist()\n        other_label.remove(max_num_label)\n        \n        amount_dic={}\n        for label in other_label:\n            num_enlarge=label_num[max_num_label]-label_num[label]\n            amount_dic[label]=num_enlarge\n            \n        return amount_dic, label_num\n            \n\n    \n    def process_arg(self, df, amount_dic, language):\n\n        parallel = Parallel(n_jobs=-1, backend=\"threading\", verbose=5)\n        \n        df_output=pd.DataFrame()\n        for eff_label, num in amount_dic.items():\n            df_perlabel=df[df['discourse_effectiveness']==eff_label]\n            rand_idx=random.sample(df_perlabel.index.values.tolist(), k=num)\n            df_perlabel_slct=df_perlabel.loc[rand_idx,:]\n            translated_data = parallel(delayed(self._one_process)(data['discourse_text'], language, i) for i, data in df_perlabel_slct.iterrows())\n            df_augmented=pd.DataFrame(translated_data)\n\n            df_perlabel_slct['index'] = df_perlabel_slct.index\n            df_argmented=pd.merge(df_augmented, df_perlabel_slct[['index', 'essay_id', 'discourse_type', 'discourse_effectiveness']], on='index', how=\"left\")\n            df_argmented['discourse_id']='augmented_data'\n            \n            df_output=pd.concat([df_output, df_argmented])\n        return df_output\n            \n    def _translate_text(self, comment, language):\n        if hasattr(comment, \"decode\"):\n            comment = comment.decode(\"utf-8\")\n        text = TextBlob(comment)\n        try:\n            text = text.translate(from_lang='en', to=language)\n            sleep(0.4)\n            text = text.translate(from_lang=language, to=\"en\")\n            sleep(0.4)\n        except NotTranslated:\n            pass\n        return str(text)\n\n    def _one_process(self, comment, language, index):\n        return {'discourse_text':self._translate_text(comment, language), 'index':index}","metadata":{"execution":{"iopub.status.busy":"2022-07-24T00:06:54.727423Z","iopub.execute_input":"2022-07-24T00:06:54.727750Z","iopub.status.idle":"2022-07-24T00:06:55.883192Z","shell.execute_reply.started":"2022-07-24T00:06:54.727696Z","shell.execute_reply":"2022-07-24T00:06:55.882114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#トレーニングデータ水増し\narg_obj=Argumentation_text([\"es\", \"de\", \"fr\"])\ndf_tr_enlarged=arg_obj.make_all_label_same_amt(dataset_tr)\ndf_tr_enlarged=df_tr_enlarged.drop(columns='index')\ndataset_tr=pd.concat([dataset_tr, df_tr_enlarged])\n#これ実行して、df同士が問題なく結合されることだけ要確認（確認事項：各ラベル数が一緒になってるか、変なnanが存在していないか）","metadata":{"execution":{"iopub.status.busy":"2022-07-24T00:06:55.885609Z","iopub.execute_input":"2022-07-24T00:06:55.886108Z","iopub.status.idle":"2022-07-24T04:21:56.812120Z","shell.execute_reply.started":"2022-07-24T00:06:55.886070Z","shell.execute_reply":"2022-07-24T04:21:56.810860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_tr.to_csv('train_arg.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:21:56.813666Z","iopub.execute_input":"2022-07-24T04:21:56.814151Z","iopub.status.idle":"2022-07-24T04:21:57.749688Z","shell.execute_reply.started":"2022-07-24T04:21:56.814106Z","shell.execute_reply":"2022-07-24T04:21:57.748568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_tr","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:23:34.965763Z","iopub.execute_input":"2022-07-24T04:23:34.966342Z","iopub.status.idle":"2022-07-24T04:23:34.997694Z","shell.execute_reply.started":"2022-07-24T04:23:34.966293Z","shell.execute_reply":"2022-07-24T04:23:34.996458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_tr['discourse_effectiveness'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:26:15.137927Z","iopub.execute_input":"2022-07-24T04:26:15.138855Z","iopub.status.idle":"2022-07-24T04:26:15.158038Z","shell.execute_reply.started":"2022-07-24T04:26:15.138799Z","shell.execute_reply":"2022-07-24T04:26:15.157217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_tr.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:26:55.271176Z","iopub.execute_input":"2022-07-24T04:26:55.271589Z","iopub.status.idle":"2022-07-24T04:26:55.320091Z","shell.execute_reply.started":"2022-07-24T04:26:55.271555Z","shell.execute_reply":"2022-07-24T04:26:55.319108Z"},"trusted":true},"execution_count":null,"outputs":[]}]}