{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n!pip install pymorphy2\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport re\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom catboost import CatBoostClassifier\n\nimport pymorphy2\nfrom nltk.tokenize import word_tokenize\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-28T10:32:44.671586Z","iopub.execute_input":"2021-05-28T10:32:44.672167Z","iopub.status.idle":"2021-05-28T10:32:59.379846Z","shell.execute_reply.started":"2021-05-28T10:32:44.672083Z","shell.execute_reply":"2021-05-28T10:32:59.378531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/Category.csv.zip')\npairs_train = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/ItemPairs_train.csv.zip')\nitem_info_train = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/ItemInfo_train.csv.zip')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:32:59.381543Z","iopub.execute_input":"2021-05-28T10:32:59.38192Z","iopub.status.idle":"2021-05-28T10:34:27.227361Z","shell.execute_reply.started":"2021-05-28T10:32:59.381885Z","shell.execute_reply":"2021-05-28T10:34:27.226302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nimport re\nimport string\nmorph = pymorphy2.MorphAnalyzer()\n\ndef del_stop_words(tokens):\n    filtered_tokens = []\n    for token in tokens:\n        if token not in stopwords.words(\"russian\"):\n            filtered_tokens.append(token)\n            \n    filtered_tokens = [x for x in filtered_tokens if not re.fullmatch('[' + string.punctuation + ']+', x)]\n    return filtered_tokens\n\ndef tokenize(text):\n    if not text:\n        print('The text to be tokenized is a None type. Defaulting to blank string.')\n        text = ''\n    return word_tokenize(text, 'russian')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.229245Z","iopub.execute_input":"2021-05-28T10:34:27.229599Z","iopub.status.idle":"2021-05-28T10:34:27.584632Z","shell.execute_reply.started":"2021-05-28T10:34:27.229534Z","shell.execute_reply":"2021-05-28T10:34:27.58383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[:1000]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.58626Z","iopub.execute_input":"2021-05-28T10:34:27.586865Z","iopub.status.idle":"2021-05-28T10:34:27.71474Z","shell.execute_reply.started":"2021-05-28T10:34:27.58682Z","shell.execute_reply":"2021-05-28T10:34:27.712807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pairs_train.merge(item_info_train[['itemID', 'title', 'description']], how='inner', left_on='itemID_1', right_on='itemID')\nX = X.rename(columns={'title':'title_1', 'description':'description_1'})\nX = X.merge(item_info_train[['itemID', 'title', 'description']], how ='inner', left_on='itemID_2', right_on='itemID')\nX = X.rename(columns={'title': 'title_2', 'description':'description_2'})\nX = X[['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate']]\nX = X.dropna()\nX_t = X[:1000]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:35.816308Z","iopub.execute_input":"2021-05-28T10:34:35.816768Z","iopub.status.idle":"2021-05-28T10:34:45.951641Z","shell.execute_reply.started":"2021-05-28T10:34:35.816735Z","shell.execute_reply":"2021-05-28T10:34:45.950803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Токенизация","metadata":{}},{"cell_type":"code","source":"X_t['title_1'] = X_t['title_1'].apply(tokenize)\nX_t['title_2'] = X_t['title_2'].apply(tokenize)\nX_t['description_1'] = X_t['description_1'].apply(tokenize)\nX_t['description_2'] = X_t['description_2'].apply(tokenize)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.717704Z","iopub.status.idle":"2021-05-28T10:34:27.718164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Удаление стоп слов","metadata":{}},{"cell_type":"code","source":"X_t['title_1'] = X_t['title_1'].apply(del_stop_words)\nX_t['title_2'] = X_t['title_2'].apply(del_stop_words)\nX_t['description_1'] = X_t['description_1'].apply(del_stop_words)\nX_t['description_2'] = X_t['description_2'].apply(del_stop_words)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.719141Z","iopub.status.idle":"2021-05-28T10:34:27.719616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Леммитизация","metadata":{}},{"cell_type":"code","source":"X_t['title_1'] = X_t['title_1'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['title_2'] = X_t['title_2'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['description_1'] = X_t['description_1'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['description_2'] = X_t['description_2'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.720573Z","iopub.status.idle":"2021-05-28T10:34:27.721007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Фитчи","metadata":{}},{"cell_type":"code","source":"X_t['title_eq'] = X_t[['title_1', 'title_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\nX_t['descr_eq'] = X_t[['description_1', 'description_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\n\nX_t['title_len1'] = X_t['title_1'].apply(lambda x: len(str(x)))\nX_t['title_len2'] = X_t['title_2'].apply(lambda x: len(str(x)))\nX_t['title_wrd1'] = X_t['title_1'].apply(lambda x: len(str(x).split()))\nX_t['title_wrd2'] = X_t['title_2'].apply(lambda x: len(str(x).split()))\n\nX_t['descr_len1'] = X_t['description_1'].apply(lambda x: len(str(x)))\nX_t['descr_len2'] = X_t['description_2'].apply(lambda x: len(str(x)))\nX_t['descr_wrd1'] = X_t['description_1'].apply(lambda x: len(str(x).split()))\nX_t['descr_wrd2'] = X_t['description_2'].apply(lambda x: len(str(x).split()))","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.721976Z","iopub.status.idle":"2021-05-28T10:34:27.722424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Анализ","metadata":{}},{"cell_type":"code","source":"X_data = X_t.drop(['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate'], axis=1)\ny = X_t['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\ncat = CatBoostClassifier()\ncat.fit(X_train, y_train)\n\ny_test_predict = cat.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.723315Z","iopub.status.idle":"2021-05-28T10:34:27.723768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hamming_distance(s1, s2):\n    return sum(ch1 != ch2 for ch1,ch2 in zip(s1,s2))\n\ndef dist(image_hash_1, image_hash_2):\n    dists = []\n    for h1 in image_hash_1:\n        for h2 in image_hash_2:\n            dists.append(hamming_distance(h1, h2))\n    \n    return min(dists)\n\ndef get_hash(s):\n    hashes_list = []\n    for id in s:\n        #if id not in skip:\n        hashes_list.append(hashes.loc[id]['image_hash'])\n        \n    return hashes_list","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.724645Z","iopub.status.idle":"2021-05-28T10:34:27.725062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashes = pd.DataFrame()\nfor n in range(10):\n    hash_n = pd.read_csv('../input/hashes/image_hash_' + str(n) +'/image_hash_'+ str(n)+ '.csv')\n    hash_n['image_id'] = hash_n['image_id'].apply(lambda x: re.sub('.+/\\d+/','',x))\n    hashes = hashes.append(hash_n)\n    \nhashes = hashes.drop(['Unnamed: 0'], axis=1)\nhashes.index = hashes.image_id\nhashes = hashes[['image_hash']]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.726069Z","iopub.status.idle":"2021-05-28T10:34:27.72651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_h = pairs_train.merge(item_info_train[['itemID', 'images_array']], how='inner', left_on='itemID_1', right_on='itemID')\nX_h = X_h.rename(columns={'title':'title_1', 'images_array':'images_array_1'})\nX_h = X_h.merge(item_info_train[['itemID', 'images_array']], how ='inner', left_on='itemID_2', right_on='itemID')\nX_h = X_h.rename(columns={'title': 'title_2', 'images_array':'images_array_2'})\nX_h = X_h[['images_array_1', 'images_array_2', 'isDuplicate']]\nX_h = X_h.dropna()\nX_h = X_h[:1000]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.727467Z","iopub.status.idle":"2021-05-28T10:34:27.727938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = X_h[['dist']]\ny = X_h['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\ncat = CatBoostClassifier()\ncat.fit(X_train, y_train)\n\ny_test_predict = cat.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.728795Z","iopub.status.idle":"2021-05-28T10:34:27.72921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_t['dist'] = X_h['dist']\nX_t = X_t.dropna()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.730198Z","iopub.status.idle":"2021-05-28T10:34:27.730682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = X_t.drop(['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate'], axis=1)\ny = X_t['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\ncat = CatBoostClassifier()\ncat.fit(X_train, y_train)\n\ny_test_predict = cat.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T10:34:27.731578Z","iopub.status.idle":"2021-05-28T10:34:27.731999Z"},"trusted":true},"execution_count":null,"outputs":[]}]}