{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"id":"bni14HV746bh","outputId":"29e70d2d-dfa8-4228-fc6d-d719456eb11b","execution":{"iopub.status.busy":"2022-07-13T11:36:30.621227Z","iopub.execute_input":"2022-07-13T11:36:30.621798Z","iopub.status.idle":"2022-07-13T11:36:30.633589Z","shell.execute_reply.started":"2022-07-13T11:36:30.621769Z","shell.execute_reply":"2022-07-13T11:36:30.631550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"tensorflow-text==2.8.*\"","metadata":{"execution":{"iopub.status.busy":"2022-07-13T11:07:18.068434Z","iopub.execute_input":"2022-07-13T11:07:18.069130Z","iopub.status.idle":"2022-07-13T11:07:29.867177Z","shell.execute_reply.started":"2022-07-13T11:07:18.069099Z","shell.execute_reply":"2022-07-13T11:07:29.865695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tf-models-official==2.7.0","metadata":{"id":"G_mXpswI5hW9","outputId":"b17c8887-a2f7-4c0e-d9ee-558d278c856b","execution":{"iopub.status.busy":"2022-07-13T11:08:22.774105Z","iopub.execute_input":"2022-07-13T11:08:22.774826Z","iopub.status.idle":"2022-07-13T11:08:35.622324Z","shell.execute_reply.started":"2022-07-13T11:08:22.774792Z","shell.execute_reply":"2022-07-13T11:08:35.620858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nimport tensorflow_text as text\nfrom official.nlp import optimization\nimport matplotlib.pyplot as plt\nimport pandas as pd","metadata":{"id":"Puvr37XJ2K8Z","execution":{"iopub.status.busy":"2022-07-13T11:09:01.926940Z","iopub.execute_input":"2022-07-13T11:09:01.927369Z","iopub.status.idle":"2022-07-13T11:09:01.933905Z","shell.execute_reply.started":"2022-07-13T11:09:01.927336Z","shell.execute_reply":"2022-07-13T11:09:01.932232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')\ntrain_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')","metadata":{"id":"uSaPWkU6WCga","execution":{"iopub.status.busy":"2022-07-13T11:10:00.749987Z","iopub.execute_input":"2022-07-13T11:10:00.750492Z","iopub.status.idle":"2022-07-13T11:10:00.793550Z","shell.execute_reply.started":"2022-07-13T11:10:00.750434Z","shell.execute_reply":"2022-07-13T11:10:00.792294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T11:10:03.172894Z","iopub.execute_input":"2022-07-13T11:10:03.173550Z","iopub.status.idle":"2022-07-13T11:10:03.205072Z","shell.execute_reply.started":"2022-07-13T11:10:03.173499Z","shell.execute_reply":"2022-07-13T11:10:03.203173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = train_data[['text', 'target']]\ntrain_ds","metadata":{"id":"K75ilxKJXmeR","outputId":"22d33fd7-7498-47ef-bf45-df532e769ce6","execution":{"iopub.status.busy":"2022-07-13T11:10:13.548299Z","iopub.execute_input":"2022-07-13T11:10:13.548679Z","iopub.status.idle":"2022-07-13T11:10:13.572990Z","shell.execute_reply.started":"2022-07-13T11:10:13.548649Z","shell.execute_reply":"2022-07-13T11:10:13.571512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8'","metadata":{"id":"gWv7fT3O5Az8","execution":{"iopub.status.busy":"2022-07-13T11:10:13.607262Z","iopub.execute_input":"2022-07-13T11:10:13.607900Z","iopub.status.idle":"2022-07-13T11:10:13.614298Z","shell.execute_reply.started":"2022-07-13T11:10:13.607870Z","shell.execute_reply":"2022-07-13T11:10:13.612941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfhub_handle_encoder = 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1'\ntfhub_handle_preprocess = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'","metadata":{"id":"EtXkYXIf6Hwl","execution":{"iopub.status.busy":"2022-07-13T11:10:13.625177Z","iopub.execute_input":"2022-07-13T11:10:13.625514Z","iopub.status.idle":"2022-07-13T11:10:13.631313Z","shell.execute_reply.started":"2022-07-13T11:10:13.625448Z","shell.execute_reply":"2022-07-13T11:10:13.630038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_model = hub.KerasLayer(tfhub_handle_encoder)","metadata":{"id":"KNqbA0n13ThK","execution":{"iopub.status.busy":"2022-07-13T11:10:13.644938Z","iopub.execute_input":"2022-07-13T11:10:13.645731Z","iopub.status.idle":"2022-07-13T11:10:25.689909Z","shell.execute_reply.started":"2022-07-13T11:10:13.645684Z","shell.execute_reply":"2022-07-13T11:10:25.688469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)","metadata":{"id":"HXEoUC2_7MJU","execution":{"iopub.status.busy":"2022-07-13T11:10:25.692553Z","iopub.execute_input":"2022-07-13T11:10:25.692912Z","iopub.status.idle":"2022-07-13T11:10:28.748593Z","shell.execute_reply.started":"2022-07-13T11:10:25.692884Z","shell.execute_reply":"2022-07-13T11:10:28.747352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_text = \"Hello I am ravikant yadav.\"","metadata":{"id":"Z2O3x44_8M13","execution":{"iopub.status.busy":"2022-07-13T11:10:28.750048Z","iopub.execute_input":"2022-07-13T11:10:28.750490Z","iopub.status.idle":"2022-07-13T11:10:28.758058Z","shell.execute_reply.started":"2022-07-13T11:10:28.750436Z","shell.execute_reply":"2022-07-13T11:10:28.756706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_text = bert_preprocess_model([test_text])","metadata":{"id":"d5SsbPWZ8ceO","execution":{"iopub.status.busy":"2022-07-13T11:10:28.762065Z","iopub.execute_input":"2022-07-13T11:10:28.762905Z","iopub.status.idle":"2022-07-13T11:10:28.979993Z","shell.execute_reply.started":"2022-07-13T11:10:28.762860Z","shell.execute_reply":"2022-07-13T11:10:28.978885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'keys :  {list(processed_text.keys())}')\nprint(f'input_word_id :  {processed_text[\"input_word_ids\"]}')\nprint(f'type_id :  {processed_text[\"input_type_ids\"]}')\nprint(f'input_mask :  {processed_text[\"input_mask\"]}')","metadata":{"id":"BSHJLGdi8jaa","outputId":"84314173-aa89-43c3-9ba1-8e74c10a4d0f","execution":{"iopub.status.busy":"2022-07-13T11:10:28.981926Z","iopub.execute_input":"2022-07-13T11:10:28.982926Z","iopub.status.idle":"2022-07-13T11:10:28.995249Z","shell.execute_reply.started":"2022-07-13T11:10:28.982880Z","shell.execute_reply":"2022-07-13T11:10:28.993534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_results = bert_model(processed_text)\n","metadata":{"id":"OGsdY8h69F31","execution":{"iopub.status.busy":"2022-07-13T11:10:28.997457Z","iopub.execute_input":"2022-07-13T11:10:28.998010Z","iopub.status.idle":"2022-07-13T11:10:30.524033Z","shell.execute_reply.started":"2022-07-13T11:10:28.997959Z","shell.execute_reply":"2022-07-13T11:10:30.522784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(bert_results.keys())","metadata":{"id":"K8i63UBYB5Ja","outputId":"77eac7fe-c53b-413c-9895-2386835b9c5f","execution":{"iopub.status.busy":"2022-07-13T11:10:30.526033Z","iopub.execute_input":"2022-07-13T11:10:30.526510Z","iopub.status.idle":"2022-07-13T11:10:30.538018Z","shell.execute_reply.started":"2022-07-13T11:10:30.526455Z","shell.execute_reply":"2022-07-13T11:10:30.536537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_classifier_model():\n  text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n  processing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocess_text')\n  encoder_input = processing_layer(text_input)\n  encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BertEncoders')\n  outputs = encoder(encoder_input)\n  net = outputs['pooled_output']\n  net = tf.keras.layers.Dropout(.1)(net)\n  net = tf.keras.layers.Dense(1, activation=None, name='classifier')(net)\n  return tf.keras.Model(text_input, net)","metadata":{"id":"HphyBi6r-C0g","execution":{"iopub.status.busy":"2022-07-13T11:10:30.539193Z","iopub.execute_input":"2022-07-13T11:10:30.539889Z","iopub.status.idle":"2022-07-13T11:10:30.590510Z","shell.execute_reply.started":"2022-07-13T11:10:30.539845Z","shell.execute_reply":"2022-07-13T11:10:30.589205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model = build_classifier_model()\n","metadata":{"id":"hpYep8mEEV0k","execution":{"iopub.status.busy":"2022-07-13T11:10:30.591827Z","iopub.execute_input":"2022-07-13T11:10:30.593701Z","iopub.status.idle":"2022-07-13T11:10:41.349778Z","shell.execute_reply.started":"2022-07-13T11:10:30.593660Z","shell.execute_reply":"2022-07-13T11:10:41.348387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(classifier_model)","metadata":{"id":"fLyhPnBtEgP2","outputId":"60faf04b-141b-4eb8-a97f-d8b6a89127b6","execution":{"iopub.status.busy":"2022-07-13T11:10:41.355841Z","iopub.execute_input":"2022-07-13T11:10:41.356217Z","iopub.status.idle":"2022-07-13T11:10:42.821037Z","shell.execute_reply.started":"2022-07-13T11:10:41.356188Z","shell.execute_reply":"2022-07-13T11:10:42.819114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\nmetrics = tf.metrics.BinaryAccuracy()","metadata":{"id":"4YMcq-ykE05X","execution":{"iopub.status.busy":"2022-07-13T11:10:42.823891Z","iopub.execute_input":"2022-07-13T11:10:42.824495Z","iopub.status.idle":"2022-07-13T11:10:42.843742Z","shell.execute_reply.started":"2022-07-13T11:10:42.824449Z","shell.execute_reply":"2022-07-13T11:10:42.842053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 5\nsteps_per_epochs = 30\nnum_train_steps = epochs * steps_per_epochs\nnum_warmup_steps = int(0.02*num_train_steps)\n\nlr = 3e-5\noptimizer = optimization.create_optimizer(init_lr = lr,\n                                          num_train_steps=num_train_steps,\n                                          num_warmup_steps=num_warmup_steps,\n                                          optimizer_type='adamw'\n                                          )","metadata":{"id":"YqK2-VWdFQOS","execution":{"iopub.status.busy":"2022-07-13T11:10:42.847729Z","iopub.execute_input":"2022-07-13T11:10:42.848666Z","iopub.status.idle":"2022-07-13T11:10:42.857596Z","shell.execute_reply.started":"2022-07-13T11:10:42.848617Z","shell.execute_reply":"2022-07-13T11:10:42.855955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model.compile(optimizer = optimizer, loss=loss, metrics=metrics)","metadata":{"id":"T6j0CKCVZ624","execution":{"iopub.status.busy":"2022-07-13T11:10:42.860522Z","iopub.execute_input":"2022-07-13T11:10:42.861163Z","iopub.status.idle":"2022-07-13T11:10:42.882529Z","shell.execute_reply.started":"2022-07-13T11:10:42.861119Z","shell.execute_reply":"2022-07-13T11:10:42.881247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"id":"oHvXkRBAa_po","execution":{"iopub.status.busy":"2022-07-13T11:10:42.884806Z","iopub.execute_input":"2022-07-13T11:10:42.885369Z","iopub.status.idle":"2022-07-13T11:10:43.027791Z","shell.execute_reply.started":"2022-07-13T11:10:42.885326Z","shell.execute_reply":"2022-07-13T11:10:43.026077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_ds.text,train_ds.target, test_size=.15, random_state=42)","metadata":{"id":"DT5MTX9if-64","execution":{"iopub.status.busy":"2022-07-13T11:10:43.032085Z","iopub.execute_input":"2022-07-13T11:10:43.032859Z","iopub.status.idle":"2022-07-13T11:10:43.048459Z","shell.execute_reply.started":"2022-07-13T11:10:43.032810Z","shell.execute_reply":"2022-07-13T11:10:43.047032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = classifier_model.fit(x=X_train,y=y_train,  validation_data = (X_test, y_test), epochs = 32)","metadata":{"id":"AaRTK90WawjL","outputId":"52eeb188-3ba9-4ecb-b400-5b725071555b","execution":{"iopub.status.busy":"2022-07-13T11:10:43.051384Z","iopub.execute_input":"2022-07-13T11:10:43.052902Z","iopub.status.idle":"2022-07-13T11:35:01.950475Z","shell.execute_reply.started":"2022-07-13T11:10:43.052867Z","shell.execute_reply":"2022-07-13T11:35:01.948995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model.evaluate(X_test,y_test)","metadata":{"id":"CSbEQRAyhIQR","outputId":"35a7fd8b-80de-4de0-f5ad-a58519f39b35","execution":{"iopub.status.busy":"2022-07-13T11:35:01.954401Z","iopub.execute_input":"2022-07-13T11:35:01.954777Z","iopub.status.idle":"2022-07-13T11:35:07.424450Z","shell.execute_reply.started":"2022-07-13T11:35:01.954747Z","shell.execute_reply":"2022-07-13T11:35:07.423076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[['text']]","metadata":{"id":"BCJ3L7IPpEXI","outputId":"6422e638-f3f5-4d02-b2ca-57a92b1baa0d","execution":{"iopub.status.busy":"2022-07-13T11:35:07.426093Z","iopub.execute_input":"2022-07-13T11:35:07.427164Z","iopub.status.idle":"2022-07-13T11:35:07.448355Z","shell.execute_reply.started":"2022-07-13T11:35:07.427103Z","shell.execute_reply":"2022-07-13T11:35:07.446751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ythat = classifier_model.predict(test_data.text)","metadata":{"id":"awsmInUqpLSJ","execution":{"iopub.status.busy":"2022-07-13T11:35:07.450780Z","iopub.execute_input":"2022-07-13T11:35:07.451393Z","iopub.status.idle":"2022-07-13T11:35:28.452760Z","shell.execute_reply.started":"2022-07-13T11:35:07.451351Z","shell.execute_reply":"2022-07-13T11:35:28.451466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ythat","metadata":{"id":"G_BUsgtqpXXh","outputId":"3f4393f2-a709-4c03-b0a3-20f13f39486f","execution":{"iopub.status.busy":"2022-07-13T11:35:28.454760Z","iopub.execute_input":"2022-07-13T11:35:28.455300Z","iopub.status.idle":"2022-07-13T11:35:28.465697Z","shell.execute_reply.started":"2022-07-13T11:35:28.455223Z","shell.execute_reply":"2022-07-13T11:35:28.464164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['pred'] = ythat","metadata":{"id":"CpUA9KEAq2pr","execution":{"iopub.status.busy":"2022-07-13T11:36:12.949842Z","iopub.execute_input":"2022-07-13T11:36:12.950292Z","iopub.status.idle":"2022-07-13T11:36:12.957276Z","shell.execute_reply.started":"2022-07-13T11:36:12.950260Z","shell.execute_reply":"2022-07-13T11:36:12.955346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['pred'].max()","metadata":{"id":"Vtd5fN_OyT5Z","outputId":"581ea14f-3420-468d-b55e-a4e14403bf25","execution":{"iopub.status.busy":"2022-07-13T11:36:09.716260Z","iopub.execute_input":"2022-07-13T11:36:09.716734Z","iopub.status.idle":"2022-07-13T11:36:09.727206Z","shell.execute_reply.started":"2022-07-13T11:36:09.716691Z","shell.execute_reply":"2022-07-13T11:36:09.725616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_class(x):\n  if x >= 0:\n    return 1\n  else:\n     return 0","metadata":{"id":"6-4fgzATyjxp","execution":{"iopub.status.busy":"2022-07-13T11:36:15.936005Z","iopub.execute_input":"2022-07-13T11:36:15.936456Z","iopub.status.idle":"2022-07-13T11:36:15.943532Z","shell.execute_reply.started":"2022-07-13T11:36:15.936426Z","shell.execute_reply":"2022-07-13T11:36:15.942010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['target'] = test_data['pred'].apply(lambda x : map_class(x))","metadata":{"id":"wm_zhK-R2mSD","execution":{"iopub.status.busy":"2022-07-13T11:36:18.410941Z","iopub.execute_input":"2022-07-13T11:36:18.411415Z","iopub.status.idle":"2022-07-13T11:36:18.422872Z","shell.execute_reply.started":"2022-07-13T11:36:18.411353Z","shell.execute_reply":"2022-07-13T11:36:18.421416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[['id', 'target']].to_csv('submission.csv', index=False)","metadata":{"id":"qPmctuKe2vpS","execution":{"iopub.status.busy":"2022-07-13T11:36:37.371295Z","iopub.execute_input":"2022-07-13T11:36:37.371678Z","iopub.status.idle":"2022-07-13T11:36:37.387899Z","shell.execute_reply.started":"2022-07-13T11:36:37.371647Z","shell.execute_reply":"2022-07-13T11:36:37.386525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Qq0_ni_G6MpU"},"execution_count":null,"outputs":[]}]}