{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ../input/avito-duplicate-ads-detection","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:25:55.833751Z","iopub.execute_input":"2022-09-06T05:25:55.834199Z","iopub.status.idle":"2022-09-06T05:25:55.884205Z","shell.execute_reply.started":"2022-09-06T05:25:55.834086Z","shell.execute_reply":"2022-09-06T05:25:55.883175Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"/kaggle/input/avito-duplicate-ads-detection\n","output_type":"stream"}]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:25:57.480613Z","iopub.execute_input":"2022-09-06T05:25:57.481368Z","iopub.status.idle":"2022-09-06T05:25:58.458711Z","shell.execute_reply.started":"2022-09-06T05:25:57.481331Z","shell.execute_reply":"2022-09-06T05:25:58.457569Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Category.csv.zip Images_5.zip\t\t ItemInfo_train.csv.zip\nImages_0.zip\t Images_6.zip\t\t ItemPairs_test.csv.zip\nImages_1.zip\t Images_7.zip\t\t ItemPairs_train.csv.zip\nImages_2.zip\t Images_8.zip\t\t Location.csv.zip\nImages_3.zip\t Images_9.zip\t\t Random_submission.csv.zip\nImages_4.zip\t ItemInfo_test.csv.zip\n","output_type":"stream"}]},{"cell_type":"code","source":"import pandas as pd\nimport zipfile\n\nzf_info_train = zipfile.ZipFile('ItemInfo_train.csv.zip') \ndf_info_train = pd.read_csv(zf_info_train.open('ItemInfo_train.csv'))\ndf_info_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:25:59.580834Z","iopub.execute_input":"2022-09-06T05:25:59.581985Z","iopub.status.idle":"2022-09-06T05:27:22.578739Z","shell.execute_reply.started":"2022-09-06T05:25:59.581935Z","shell.execute_reply":"2022-09-06T05:27:22.577824Z"},"trusted":true},"execution_count":3,"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":" itemID categoryID title \\\n0 1 81 Продам Камаз 6520 \n1 3 14 Yamaha r6 \n2 4 84 iPhone 3gs 8gb \n3 7 84 Xiaomi Mi4 3гб RAM + 16гб ROM белый \n4 8 39 Лыжные ботинки \n\n description \\\n0 Продам Камаз 6520 20 тонн \n1 Весь в тюнинге. \n2 Телефон в хорошем состоянии, трещин и сколов н... \n3 Отличный подарок на новый год от \"китайской ap... \n4 Лыжные ботинки в хорошем состоянии, 34 размер \n\n images_array \\\n0 1064094, 5252822, 6645873, 6960145, 9230265 \n1 11919573, 14412228, 3204180, 6646877 \n2 14384831, 6102021 \n3 NaN \n4 13718854, 4787310 \n\n attrsJSON price locationID \\\n0 {\"Вид техники\":\"Грузовики\"} 300000.0 648140 \n1 {\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С... 300000.0 639040 \n2 {\"Вид телефона\":\"iPhone\"} 3500.0 640650 \n3 {\"Вид телефона\":\"Другие марки\"} 13500.0 662210 \n4 {\"Вид товара\":\"Зимние виды спорта\"} 500.0 624360 \n\n metroID lat lon \n0 NaN 64.686946 30.815924 \n1 NaN 55.678037 37.256548 \n2 NaN 56.239398 43.460458 \n3 NaN 55.777170 37.586194 \n4 NaN 55.777170 37.586194 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
itemIDcategoryIDtitledescriptionimages_arrayattrsJSONpricelocationIDmetroIDlatlon
0181Продам Камаз 6520Продам Камаз 6520 20 тонн1064094, 5252822, 6645873, 6960145, 9230265{\"Вид техники\":\"Грузовики\"}300000.0648140NaN64.68694630.815924
1314Yamaha r6Весь в тюнинге.11919573, 14412228, 3204180, 6646877{\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С...300000.0639040NaN55.67803737.256548
2484iPhone 3gs 8gbТелефон в хорошем состоянии, трещин и сколов н...14384831, 6102021{\"Вид телефона\":\"iPhone\"}3500.0640650NaN56.23939843.460458
3784Xiaomi Mi4 3гб RAM + 16гб ROM белыйОтличный подарок на новый год от \"китайской ap...NaN{\"Вид телефона\":\"Другие марки\"}13500.0662210NaN55.77717037.586194
4839Лыжные ботинкиЛыжные ботинки в хорошем состоянии, 34 размер13718854, 4787310{\"Вид товара\":\"Зимние виды спорта\"}500.0624360NaN55.77717037.586194
\n
"},"metadata":{}}]},{"cell_type":"code","source":"import pandas as pd\nimport zipfile\n\nzf_pairs_train = zipfile.ZipFile('ItemPairs_train.csv.zip') \ndf_pairs_train = pd.read_csv(zf_pairs_train.open('ItemPairs_train.csv'))\ndf_pairs_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:22.580645Z","iopub.execute_input":"2022-09-06T05:27:22.581015Z","iopub.status.idle":"2022-09-06T05:27:23.747029Z","shell.execute_reply.started":"2022-09-06T05:27:22.58098Z","shell.execute_reply":"2022-09-06T05:27:23.74602Z"},"trusted":true},"execution_count":4,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":" itemID_1 itemID_2 isDuplicate generationMethod\n0 1 4112648 1 1\n1 3 1991275 1 1\n2 4 1223296 0 1\n3 7 1058851 1 1\n4 8 2161930 1 1","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
itemID_1itemID_2isDuplicategenerationMethod
01411264811
13199127511
24122329601
37105885111
48216193011
\n
"},"metadata":{}}]},{"cell_type":"code","source":"# title1 = []\n# title2 = []\n# price1 = []\n# price2 = []\n# for index1, row1 in df_pairs_train.iterrows():\n# for index2, row2 in df_info_train.iterrows():\n# if row1['itemID_1'] == row2['itemID']:\n# title1.append(row2['title'])\n# price1.append(row2['price'])\n# elif row1['itemID_2'] == row2['itemID']:\n# title2.append(row2['title'])\n# price2.append(row2['price'])\n ","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:48.33052Z","iopub.execute_input":"2022-09-06T05:27:48.330895Z","iopub.status.idle":"2022-09-06T05:27:48.335598Z","shell.execute_reply.started":"2022-09-06T05:27:48.330865Z","shell.execute_reply":"2022-09-06T05:27:48.334497Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# df1 = {}\n# df1 = pd.DataFrame(df1)\n# df1['item_1'] = [1, 2, 3, 4, 5]\n# df1['item_2'] = [10, 7, 8, 9, 6]\n# df1['is_duplicate'] = [0, 1, 1, 0, 1]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:48.676457Z","iopub.execute_input":"2022-09-06T05:27:48.677119Z","iopub.status.idle":"2022-09-06T05:27:48.681562Z","shell.execute_reply.started":"2022-09-06T05:27:48.677067Z","shell.execute_reply":"2022-09-06T05:27:48.680466Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"# df2 = {}\n# df2 = pd.DataFrame(df2)\n# df2['item_id'] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n# df2['title'] = [\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\"]\n# df2['price'] = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:49.035471Z","iopub.execute_input":"2022-09-06T05:27:49.036408Z","iopub.status.idle":"2022-09-06T05:27:49.041446Z","shell.execute_reply.started":"2022-09-06T05:27:49.036366Z","shell.execute_reply":"2022-09-06T05:27:49.040444Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"# df1.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:49.366543Z","iopub.execute_input":"2022-09-06T05:27:49.367646Z","iopub.status.idle":"2022-09-06T05:27:49.372526Z","shell.execute_reply.started":"2022-09-06T05:27:49.367599Z","shell.execute_reply":"2022-09-06T05:27:49.371348Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"# df2.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:49.742124Z","iopub.execute_input":"2022-09-06T05:27:49.743624Z","iopub.status.idle":"2022-09-06T05:27:49.747909Z","shell.execute_reply.started":"2022-09-06T05:27:49.743583Z","shell.execute_reply":"2022-09-06T05:27:49.746775Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"# df3 = pd.merge(df1, df2, how='inner', on=None, left_on=df1[\"item_2\"], right_on=df2[\"item_id\"])\n# df3 = pd.merge(df1, df2, how='outer', on=[df2['item_id'], df1['item_1']])\n\ndf3 = pd.merge(df_pairs_train, df_info_train, left_on=df_pairs_train[\"itemID_1\"], right_on=df_info_train['itemID'], how='inner')\ndf4 = pd.merge(df_pairs_train, df_info_train, left_on=df_pairs_train[\"itemID_2\"], right_on=df_info_train['itemID'], how='inner')","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:51.016796Z","iopub.execute_input":"2022-09-06T05:27:51.017176Z","iopub.status.idle":"2022-09-06T05:27:55.563364Z","shell.execute_reply.started":"2022-09-06T05:27:51.017143Z","shell.execute_reply":"2022-09-06T05:27:55.562134Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"df3.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:56.183084Z","iopub.execute_input":"2022-09-06T05:27:56.183803Z","iopub.status.idle":"2022-09-06T05:27:56.205749Z","shell.execute_reply.started":"2022-09-06T05:27:56.183766Z","shell.execute_reply":"2022-09-06T05:27:56.204817Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":" key_0 itemID_1 itemID_2 isDuplicate generationMethod itemID \\\n0 1 1 4112648 1 1 1 \n1 3 3 1991275 1 1 3 \n2 4 4 1223296 0 1 4 \n3 7 7 1058851 1 1 7 \n4 8 8 2161930 1 1 8 \n\n categoryID title \\\n0 81 Продам Камаз 6520 \n1 14 Yamaha r6 \n2 84 iPhone 3gs 8gb \n3 84 Xiaomi Mi4 3гб RAM + 16гб ROM белый \n4 39 Лыжные ботинки \n\n description \\\n0 Продам Камаз 6520 20 тонн \n1 Весь в тюнинге. \n2 Телефон в хорошем состоянии, трещин и сколов н... \n3 Отличный подарок на новый год от \"китайской ap... \n4 Лыжные ботинки в хорошем состоянии, 34 размер \n\n images_array \\\n0 1064094, 5252822, 6645873, 6960145, 9230265 \n1 11919573, 14412228, 3204180, 6646877 \n2 14384831, 6102021 \n3 NaN \n4 13718854, 4787310 \n\n attrsJSON price locationID \\\n0 {\"Вид техники\":\"Грузовики\"} 300000.0 648140 \n1 {\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С... 300000.0 639040 \n2 {\"Вид телефона\":\"iPhone\"} 3500.0 640650 \n3 {\"Вид телефона\":\"Другие марки\"} 13500.0 662210 \n4 {\"Вид товара\":\"Зимние виды спорта\"} 500.0 624360 \n\n metroID lat lon \n0 NaN 64.686946 30.815924 \n1 NaN 55.678037 37.256548 \n2 NaN 56.239398 43.460458 \n3 NaN 55.777170 37.586194 \n4 NaN 55.777170 37.586194 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
key_0itemID_1itemID_2isDuplicategenerationMethoditemIDcategoryIDtitledescriptionimages_arrayattrsJSONpricelocationIDmetroIDlatlon
011411264811181Продам Камаз 6520Продам Камаз 6520 20 тонн1064094, 5252822, 6645873, 6960145, 9230265{\"Вид техники\":\"Грузовики\"}300000.0648140NaN64.68694630.815924
133199127511314Yamaha r6Весь в тюнинге.11919573, 14412228, 3204180, 6646877{\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С...300000.0639040NaN55.67803737.256548
244122329601484iPhone 3gs 8gbТелефон в хорошем состоянии, трещин и сколов н...14384831, 6102021{\"Вид телефона\":\"iPhone\"}3500.0640650NaN56.23939843.460458
377105885111784Xiaomi Mi4 3гб RAM + 16гб ROM белыйОтличный подарок на новый год от \"китайской ap...NaN{\"Вид телефона\":\"Другие марки\"}13500.0662210NaN55.77717037.586194
488216193011839Лыжные ботинкиЛыжные ботинки в хорошем состоянии, 34 размер13718854, 4787310{\"Вид товара\":\"Зимние виды спорта\"}500.0624360NaN55.77717037.586194
\n
"},"metadata":{}}]},{"cell_type":"code","source":"df4.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:27:59.358044Z","iopub.execute_input":"2022-09-06T05:27:59.359051Z","iopub.status.idle":"2022-09-06T05:27:59.377402Z","shell.execute_reply.started":"2022-09-06T05:27:59.359015Z","shell.execute_reply":"2022-09-06T05:27:59.376271Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":" key_0 itemID_1 itemID_2 isDuplicate generationMethod itemID \\\n0 4112648 1 4112648 1 1 4112648 \n1 1991275 3 1991275 1 1 1991275 \n2 1223296 4 1223296 0 1 1223296 \n3 1058851 7 1058851 1 1 1058851 \n4 2161930 8 2161930 1 1 2161930 \n\n categoryID title \\\n0 81 Продам Камаз 6520 \n1 14 Yamaha R6 \n2 84 iPhone 3gs 32gb \n3 84 Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белый \n4 39 Ботинки для лыж \n\n description \\\n0 Продам Камаз 6520 20 тонн \n1 Весь в тюнинге. Возможен торг \n2 В отличном состоянии, в комплекте зарядник и к... \n3 Отличный подарок на новый год от \"китайской ap... \n4 Ботинки 34 размер, в хорошем состоянии \n\n images_array \\\n0 1227519, 1374615, 7072137, 8671835 \n1 11068709, 13325040, 13783238, 206652, 9458537 \n2 5709245 \n3 7900519 \n4 10053682, 986143 \n\n attrsJSON price locationID \\\n0 {\"Вид техники\":\"Грузовики\"} 300000.0 648140 \n1 {\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С... 330000.0 639040 \n2 {\"Вид телефона\":\"iPhone\"} 3500.0 640650 \n3 {\"Вид телефона\":\"Другие марки\"} 13500.0 662210 \n4 {\"Вид товара\":\"Зимние виды спорта\"} 600.0 624360 \n\n metroID lat lon \n0 NaN 64.686946 30.815924 \n1 NaN 55.678037 37.256548 \n2 NaN 56.239398 43.460458 \n3 NaN 56.135459 47.235484 \n4 NaN 55.777170 37.586194 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
key_0itemID_1itemID_2isDuplicategenerationMethoditemIDcategoryIDtitledescriptionimages_arrayattrsJSONpricelocationIDmetroIDlatlon
041126481411264811411264881Продам Камаз 6520Продам Камаз 6520 20 тонн1227519, 1374615, 7072137, 8671835{\"Вид техники\":\"Грузовики\"}300000.0648140NaN64.68694630.815924
119912753199127511199127514Yamaha R6Весь в тюнинге. Возможен торг11068709, 13325040, 13783238, 206652, 9458537{\"Вид техники\":\"Мотоциклы\", \"Вид мотоцикла\":\"С...330000.0639040NaN55.67803737.256548
212232964122329601122329684iPhone 3gs 32gbВ отличном состоянии, в комплекте зарядник и к...5709245{\"Вид телефона\":\"iPhone\"}3500.0640650NaN56.23939843.460458
310588517105885111105885184Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белыйОтличный подарок на новый год от \"китайской ap...7900519{\"Вид телефона\":\"Другие марки\"}13500.0662210NaN56.13545947.235484
421619308216193011216193039Ботинки для лыжБотинки 34 размер, в хорошем состоянии10053682, 986143{\"Вид товара\":\"Зимние виды спорта\"}600.0624360NaN55.77717037.586194
\n
"},"metadata":{}}]},{"cell_type":"code","source":"# df3_mod = df3.dropna()\ndf3_mod = df3.drop(['itemID_1', 'itemID_2', 'key_0', 'generationMethod', 'categoryID', 'description', 'images_array', 'attrsJSON', 'locationID', 'metroID', 'lat', 'lon'], axis=1)\ndf3_mod = df3_mod.rename(columns={'itemID': 'itemID_1', 'is_duplicate': 'is_duplicate1', 'price': 'price1', 'title': 'title1'})\ndf3_mod","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:02.042959Z","iopub.execute_input":"2022-09-06T05:28:02.043333Z","iopub.status.idle":"2022-09-06T05:28:02.434884Z","shell.execute_reply.started":"2022-09-06T05:28:02.043299Z","shell.execute_reply":"2022-09-06T05:28:02.433792Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":" isDuplicate itemID_1 \\\n0 1 1 \n1 1 3 \n2 0 4 \n3 1 7 \n4 1 8 \n... ... ... \n2991391 0 6104033 \n2991392 1 6104879 \n2991393 0 6105178 \n2991394 1 6106239 \n2991395 0 6106742 \n\n title1 price1 \n0 Продам Камаз 6520 300000.0 \n1 Yamaha r6 300000.0 \n2 iPhone 3gs 8gb 3500.0 \n3 Xiaomi Mi4 3гб RAM + 16гб ROM белый 13500.0 \n4 Лыжные ботинки 500.0 \n... ... ... \n2991391 Volkswagen Tiguan, 2015 1499000.0 \n2991392 Редкие банкноты СССР идт 1000.0 \n2991393 Забавная Kykлa Reborn - Катюша 8990.0 \n2991394 Nikon D3100 12000.0 \n2991395 Крестовина кардана большая Hagglunds BV-206 Лось 7000.0 \n\n[2991396 rows x 4 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
isDuplicateitemID_1title1price1
011Продам Камаз 6520300000.0
113Yamaha r6300000.0
204iPhone 3gs 8gb3500.0
317Xiaomi Mi4 3гб RAM + 16гб ROM белый13500.0
418Лыжные ботинки500.0
...............
299139106104033Volkswagen Tiguan, 20151499000.0
299139216104879Редкие банкноты СССР идт1000.0
299139306105178Забавная Kykлa Reborn - Катюша8990.0
299139416106239Nikon D310012000.0
299139506106742Крестовина кардана большая Hagglunds BV-206 Лось7000.0
\n

2991396 rows × 4 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"df3_mod.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:05.498703Z","iopub.execute_input":"2022-09-06T05:28:05.499831Z","iopub.status.idle":"2022-09-06T05:28:05.512382Z","shell.execute_reply.started":"2022-09-06T05:28:05.499783Z","shell.execute_reply":"2022-09-06T05:28:05.511265Z"},"trusted":true},"execution_count":17,"outputs":[{"execution_count":17,"output_type":"execute_result","data":{"text/plain":" isDuplicate itemID_1 title1 price1\n0 1 1 Продам Камаз 6520 300000.0\n1 1 3 Yamaha r6 300000.0\n2 0 4 iPhone 3gs 8gb 3500.0\n3 1 7 Xiaomi Mi4 3гб RAM + 16гб ROM белый 13500.0\n4 1 8 Лыжные ботинки 500.0","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
isDuplicateitemID_1title1price1
011Продам Камаз 6520300000.0
113Yamaha r6300000.0
204iPhone 3gs 8gb3500.0
317Xiaomi Mi4 3гб RAM + 16гб ROM белый13500.0
418Лыжные ботинки500.0
\n
"},"metadata":{}}]},{"cell_type":"code","source":"# df4_mod = df4.dropna()\ndf4_mod = df4.drop(['itemID_1', 'itemID_2', 'key_0', 'generationMethod', 'categoryID', 'description', 'images_array', 'attrsJSON', 'locationID', 'metroID', 'lat', 'lon'], axis=1)\ndf4_mod = df4_mod.rename(columns={'itemID': 'itemID_2', 'isDuplicate': 'is_duplicate1', 'price': 'price2', 'title': 'title2'})\ndf4_mod","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:11.917869Z","iopub.execute_input":"2022-09-06T05:28:11.918566Z","iopub.status.idle":"2022-09-06T05:28:12.374038Z","shell.execute_reply.started":"2022-09-06T05:28:11.91853Z","shell.execute_reply":"2022-09-06T05:28:12.373086Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":" is_duplicate1 itemID_2 \\\n0 1 4112648 \n1 1 1991275 \n2 0 1223296 \n3 1 1058851 \n4 1 2161930 \n... ... ... \n2991391 0 6101171 \n2991392 0 6105511 \n2991393 0 6109873 \n2991394 1 6111227 \n2991395 1 6109219 \n\n title2 price2 \n0 Продам Камаз 6520 300000.0 \n1 Yamaha R6 330000.0 \n2 iPhone 3gs 32gb 3500.0 \n3 Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белый 13500.0 \n4 Ботинки для лыж 600.0 \n... ... ... \n2991391 Крыша ВАЗ 2110 NaN \n2991392 Охранник, администратор 1600.0 \n2991393 2-к квартира, 46 м², 1/9 эт. 2600000.0 \n2991394 Редкие банкноты NaN \n2991395 Nikon D3100 15000.0 \n\n[2991396 rows x 4 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
is_duplicate1itemID_2title2price2
014112648Продам Камаз 6520300000.0
111991275Yamaha R6330000.0
201223296iPhone 3gs 32gb3500.0
311058851Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белый13500.0
412161930Ботинки для лыж600.0
...............
299139106101171Крыша ВАЗ 2110NaN
299139206105511Охранник, администратор1600.0
2991393061098732-к квартира, 46 м², 1/9 эт.2600000.0
299139416111227Редкие банкнотыNaN
299139516109219Nikon D310015000.0
\n

2991396 rows × 4 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"result = pd.concat([df3_mod, df4_mod], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:15.347516Z","iopub.execute_input":"2022-09-06T05:28:15.347872Z","iopub.status.idle":"2022-09-06T05:28:15.518439Z","shell.execute_reply.started":"2022-09-06T05:28:15.347844Z","shell.execute_reply":"2022-09-06T05:28:15.517487Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"result\nresult_mod = result.drop(['is_duplicate1'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:16.717861Z","iopub.execute_input":"2022-09-06T05:28:16.71836Z","iopub.status.idle":"2022-09-06T05:28:17.365638Z","shell.execute_reply.started":"2022-09-06T05:28:16.718324Z","shell.execute_reply":"2022-09-06T05:28:17.364664Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"result_mod","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:18.738214Z","iopub.execute_input":"2022-09-06T05:28:18.738909Z","iopub.status.idle":"2022-09-06T05:28:18.759068Z","shell.execute_reply.started":"2022-09-06T05:28:18.738874Z","shell.execute_reply":"2022-09-06T05:28:18.758035Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":" isDuplicate itemID_1 \\\n0 1 1 \n1 1 3 \n2 0 4 \n3 1 7 \n4 1 8 \n... ... ... \n2991391 0 6104033 \n2991392 1 6104879 \n2991393 0 6105178 \n2991394 1 6106239 \n2991395 0 6106742 \n\n title1 price1 \\\n0 Продам Камаз 6520 300000.0 \n1 Yamaha r6 300000.0 \n2 iPhone 3gs 8gb 3500.0 \n3 Xiaomi Mi4 3гб RAM + 16гб ROM белый 13500.0 \n4 Лыжные ботинки 500.0 \n... ... ... \n2991391 Volkswagen Tiguan, 2015 1499000.0 \n2991392 Редкие банкноты СССР идт 1000.0 \n2991393 Забавная Kykлa Reborn - Катюша 8990.0 \n2991394 Nikon D3100 12000.0 \n2991395 Крестовина кардана большая Hagglunds BV-206 Лось 7000.0 \n\n itemID_2 title2 price2 \n0 4112648 Продам Камаз 6520 300000.0 \n1 1991275 Yamaha R6 330000.0 \n2 1223296 iPhone 3gs 32gb 3500.0 \n3 1058851 Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белый 13500.0 \n4 2161930 Ботинки для лыж 600.0 \n... ... ... ... \n2991391 6101171 Крыша ВАЗ 2110 NaN \n2991392 6105511 Охранник, администратор 1600.0 \n2991393 6109873 2-к квартира, 46 м², 1/9 эт. 2600000.0 \n2991394 6111227 Редкие банкноты NaN \n2991395 6109219 Nikon D3100 15000.0 \n\n[2991396 rows x 7 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
isDuplicateitemID_1title1price1itemID_2title2price2
011Продам Камаз 6520300000.04112648Продам Камаз 6520300000.0
113Yamaha r6300000.01991275Yamaha R6330000.0
204iPhone 3gs 8gb3500.01223296iPhone 3gs 32gb3500.0
317Xiaomi Mi4 3гб RAM + 16гб ROM белый13500.01058851Смартфон Xiaomi Mi4 3гб RAM + 16гб ROM белый13500.0
418Лыжные ботинки500.02161930Ботинки для лыж600.0
........................
299139106104033Volkswagen Tiguan, 20151499000.06101171Крыша ВАЗ 2110NaN
299139216104879Редкие банкноты СССР идт1000.06105511Охранник, администратор1600.0
299139306105178Забавная Kykлa Reborn - Катюша8990.061098732-к квартира, 46 м², 1/9 эт.2600000.0
299139416106239Nikon D310012000.06111227Редкие банкнотыNaN
299139506106742Крестовина кардана большая Hagglunds BV-206 Лось7000.06109219Nikon D310015000.0
\n

2991396 rows × 7 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"!pip install -U sentence-transformers","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:21.238933Z","iopub.execute_input":"2022-09-06T05:28:21.239616Z","iopub.status.idle":"2022-09-06T05:28:35.014535Z","shell.execute_reply.started":"2022-09-06T05:28:21.23958Z","shell.execute_reply":"2022-09-06T05:28:35.013385Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"Collecting sentence-transformers\n Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m295.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.6.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.20.1)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.64.0)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.11.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.12.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.21.6)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.0.2)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.7.3)\nRequirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (3.7)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.1.97)\nRequirement already satisfied: huggingface-hub>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.8.1)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.12.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2.28.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.7.1)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (6.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.3.0)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2021.11.10)\nRequirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /opt/conda/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.12.1)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from nltk->sentence-transformers) (1.0.1)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from nltk->sentence-transformers) (8.0.4)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (3.1.0)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.7/site-packages (from torchvision->sentence-transformers) (9.1.1)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.9->huggingface-hub>=0.4.0->sentence-transformers) (3.0.9)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->huggingface-hub>=0.4.0->sentence-transformers) (3.8.0)\nRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2.1.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2022.6.15)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.3)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (1.26.12)\nBuilding wheels for collected packages: sentence-transformers\n Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125938 sha256=287b645813f4aebd9c88eaa42b7430256236ead12583206321f5e71c2a8b5ffe\n Stored in directory: /root/.cache/pip/wheels/bf/06/fb/d59c1e5bd1dac7f6cf61ec0036cc3a10ab8fecaa6b2c3d3ee9\nSuccessfully built sentence-transformers\nInstalling collected packages: sentence-transformers\nSuccessfully installed sentence-transformers-2.2.2\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\nsentences = [\"This is an example sentence\", \"Each sentence is converted\"]\n\nmodel = SentenceTransformer('sentence-transformers/distiluse-base-multilingual-cased-v2')\nembeddings = model.encode(sentences)\nprint(embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T05:28:53.043351Z","iopub.execute_input":"2022-09-06T05:28:53.043748Z","iopub.status.idle":"2022-09-06T05:30:01.615382Z","shell.execute_reply.started":"2022-09-06T05:28:53.043713Z","shell.execute_reply":"2022-09-06T05:30:01.614246Z"},"trusted":true},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading: 0%| | 0.00/690 [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
itemID_1title1price1itemID_2title2price2
27380934322062Модуль, 3.5\", HTC Desire 200 черный, 480x3201660.04180551Машинка Indesit все рабочее без дефектов чистая5500.0
21699232891520iPhone 5, 16 g12000.04702162Термокружка1070.0
16403141991385Лобзик De-Ford 500w сост отл1000.03218474Ксенон, би-ксенон1150.0
22171482986135Аренда авто с выкупомNaN3668457Зарядное устройство для р/т Senao SN-358B500.0
20447682658042Адидас800.03200789Продам мотор daewoo nexia, chevrolet lanos, ch...20000.0
.....................
16927432071027Жилетка adidas разминка2000.01949453Гальяно Б/у3000.0
23563303281740Золотая девочкаNaN3374530Ищу работуNaN
22290843010903Ресивер обмен. продажа4000.04212386Люксовая высокая из ткани и экокожи 3 в 115000.0
27683074432879Абонемент в фитнес клуб Оранж10500.06091309Запчасти для honda CR-V 3 / б/уNaN
22191102989899Системный блок AMD Phenom X3 8250/ddrii 4Gb9000.03341591Комплекты сигнализаций для защиты техники2000.0
\n

2393116 rows × 6 columns

\n"},"metadata":{}}]},{"cell_type":"code","source":"# def preprocess_text(sen):\n\n# # Remove punctuations and numbers\n# sentence = re.sub('[^a-zA-Z]', ' ', sen)\n\n# # Single character removal\n# sentence = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', sentence)\n\n# # Removing multiple spaces\n# sentence = re.sub(r'\\s+', ' ', sentence)\n\n# return sentence","metadata":{"execution":{"iopub.status.busy":"2022-09-05T20:52:05.581871Z","iopub.execute_input":"2022-09-05T20:52:05.582287Z","iopub.status.idle":"2022-09-05T20:52:05.588458Z","shell.execute_reply.started":"2022-09-05T20:52:05.582249Z","shell.execute_reply":"2022-09-05T20:52:05.587423Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"code","source":"X1_train_title1 = []\nX1_train_title1 = list(X_train[\"title1\"])\nX1_train_title2 = []\nX1_train_title2 = list(X_train[\"title2\"])\n# for sen in sentences:\n# X1_train.append(preprocess_text(sen))","metadata":{"execution":{"iopub.status.busy":"2022-09-05T21:10:50.919022Z","iopub.execute_input":"2022-09-05T21:10:50.919974Z","iopub.status.idle":"2022-09-05T21:10:51.76211Z","shell.execute_reply.started":"2022-09-05T21:10:50.919933Z","shell.execute_reply":"2022-09-05T21:10:51.761037Z"},"trusted":true},"execution_count":61,"outputs":[]},{"cell_type":"code","source":"X1_test_title1 = []\nX1_test_title1 = list(X_test[\"title1\"])\nX1_test_title2 = []\nX1_test_title2 = list(X_test[\"title2\"])\n# for sen in sentences:\n# X1_test.append(preprocess_text(sen))","metadata":{"execution":{"iopub.status.busy":"2022-09-05T21:10:53.244475Z","iopub.execute_input":"2022-09-05T21:10:53.245259Z","iopub.status.idle":"2022-09-05T21:10:53.467256Z","shell.execute_reply.started":"2022-09-05T21:10:53.245216Z","shell.execute_reply":"2022-09-05T21:10:53.466022Z"},"trusted":true},"execution_count":62,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=5000)\ntokenizer.fit_on_texts(X1_train)\n\nX1_train_title1 = tokenizer.texts_to_sequences(str(X1_train_title1))\nX1_train_title2 = tokenizer.texts_to_sequences(X1_train_title2)\n\nX1_test_title1 = tokenizer.texts_to_sequences(X1_test_title1)\nX1_test_title2 = tokenizer.texts_to_sequences(X1_test_title2)\n\n\nvocab_size = len(tokenizer.word_index) + 1\n\nmaxlen = 200\n\nX1_train = pad_sequences(X1_train, padding='post', maxlen=maxlen)\nX1_test = pad_sequences(X1_test, padding='post', maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T21:26:07.502401Z","iopub.execute_input":"2022-09-05T21:26:07.502826Z","iopub.status.idle":"2022-09-05T21:26:07.522753Z","shell.execute_reply.started":"2022-09-05T21:26:07.502782Z","shell.execute_reply":"2022-09-05T21:26:07.520734Z"},"trusted":true},"execution_count":2,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_5408/4088225912.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_on_texts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX1_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mX1_train_title1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtexts_to_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX1_train_title1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mX1_train_title2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtexts_to_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX1_train_title2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'Tokenizer' is not defined"],"ename":"NameError","evalue":"name 'Tokenizer' is not defined","output_type":"error"}]},{"cell_type":"code","source":"X1_train","metadata":{"execution":{"iopub.status.busy":"2022-09-05T21:26:03.592534Z","iopub.execute_input":"2022-09-05T21:26:03.59312Z","iopub.status.idle":"2022-09-05T21:26:03.711989Z","shell.execute_reply.started":"2022-09-05T21:26:03.592996Z","shell.execute_reply":"2022-09-05T21:26:03.70969Z"},"trusted":true},"execution_count":1,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_5408/453107565.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX1_train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'X1_train' is not defined"],"ename":"NameError","evalue":"name 'X1_train' is not defined","output_type":"error"}]}]}