{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-08T19:28:25.755446Z","iopub.execute_input":"2022-07-08T19:28:25.755867Z","iopub.status.idle":"2022-07-08T19:28:25.765965Z","shell.execute_reply.started":"2022-07-08T19:28:25.755831Z","shell.execute_reply":"2022-07-08T19:28:25.764809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport time\nfrom tqdm import tqdm\nimport seaborn as sns\nimport numpy as np\nfrom textblob import TextBlob\nimport matplotlib.pyplot as plt\nfrom sentence_transformers import SentenceTransformer\nimport faiss\nimport numpy.linalg as la\nimport warnings\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', 50)\npd.set_option('max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:43:58.404130Z","iopub.execute_input":"2022-07-08T19:43:58.404805Z","iopub.status.idle":"2022-07-08T19:43:58.411853Z","shell.execute_reply.started":"2022-07-08T19:43:58.404772Z","shell.execute_reply":"2022-07-08T19:43:58.411046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/quora-question-pairs/train.csv.zip')","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:35:21.028721Z","iopub.execute_input":"2022-07-08T19:35:21.029378Z","iopub.status.idle":"2022-07-08T19:35:23.489269Z","shell.execute_reply.started":"2022-07-08T19:35:21.029326Z","shell.execute_reply":"2022-07-08T19:35:23.488057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:41:03.591435Z","iopub.execute_input":"2022-07-08T19:41:03.592222Z","iopub.status.idle":"2022-07-08T19:41:03.607051Z","shell.execute_reply.started":"2022-07-08T19:41:03.592184Z","shell.execute_reply":"2022-07-08T19:41:03.605609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stops = set(stopwords.words('english'))\nimport re\n\ndef clean_text(text):\n    ''' Pre process and convert texts to a list of words '''\n    text = str(text)\n    text = text.lower()\n\n    # Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"can't\", \"cannot \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n\n#     text = text.split()\n\n    return text\n\ntrain['question1'] = train.question1.apply(lambda q: clean_text(q))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:48:49.073095Z","iopub.execute_input":"2022-07-08T19:48:49.073937Z","iopub.status.idle":"2022-07-08T19:49:09.840453Z","shell.execute_reply.started":"2022-07-08T19:48:49.073899Z","shell.execute_reply":"2022-07-08T19:49:09.839215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['question2'] = train.question2.apply(lambda q: clean_text(q))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:49:21.193447Z","iopub.execute_input":"2022-07-08T19:49:21.193906Z","iopub.status.idle":"2022-07-08T19:49:42.299511Z","shell.execute_reply.started":"2022-07-08T19:49:21.193858Z","shell.execute_reply":"2022-07-08T19:49:42.298095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:49:42.301872Z","iopub.execute_input":"2022-07-08T19:49:42.302759Z","iopub.status.idle":"2022-07-08T19:49:42.319114Z","shell.execute_reply.started":"2022-07-08T19:49:42.302709Z","shell.execute_reply":"2022-07-08T19:49:42.317886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['doc_len'] = train['question1'].apply(lambda words: len(words.split()))\nmax_seq_len = np.round(train['doc_len'].mean())# + 2*df['doc_len'].std()).astype(int)  # can use 2 * std\nsns.distplot(train['doc_len'], hist=True, kde=True, color='b', label='doc len')\nplt.axvline(x=max_seq_len, color='k', linestyle='--', label='avg_len + std')\nplt.title('plot length'); plt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:50:28.857351Z","iopub.execute_input":"2022-07-08T19:50:28.857815Z","iopub.status.idle":"2022-07-08T19:50:31.427783Z","shell.execute_reply.started":"2022-07-08T19:50:28.857780Z","shell.execute_reply":"2022-07-08T19:50:31.426621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:51:29.604053Z","iopub.execute_input":"2022-07-08T19:51:29.604517Z","iopub.status.idle":"2022-07-08T19:51:29.613816Z","shell.execute_reply.started":"2022-07-08T19:51:29.604482Z","shell.execute_reply":"2022-07-08T19:51:29.612554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dropna(inplace=True)\ntrain.drop_duplicates(subset=['question1', 'question2'],inplace=True)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:51:35.954744Z","iopub.execute_input":"2022-07-08T19:51:35.955237Z","iopub.status.idle":"2022-07-08T19:51:36.576571Z","shell.execute_reply.started":"2022-07-08T19:51:35.955201Z","shell.execute_reply":"2022-07-08T19:51:36.575197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-08T19:29:05.029109Z","iopub.execute_input":"2022-07-08T19:29:05.029983Z","iopub.status.idle":"2022-07-08T19:29:05.805642Z","shell.execute_reply.started":"2022-07-08T19:29:05.029916Z","shell.execute_reply":"2022-07-08T19:29:05.804497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}