{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# X = pairs_train.merge(item_info_train[['itemID', 'title', 'description']], how='inner', left_on='itemID_1', right_on='itemID')\n# X = X.rename(columns={'title':'title_1', 'description':'description_1'})\n# X = X.merge(item_info_train[['itemID', 'title', 'description']], how ='inner', left_on='itemID_2', right_on='itemID')\n# X = X.rename(columns={'title': 'title_2', 'description':'description_2'})\n# X = X[['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate']]\n# X = X.dropna()\n\n# X['title_1'] = X['title_1'].str.lower()\n# X['title_2'] = X['title_2'].str.lower()\n\n# X['description_1'] = X['description_1'].str.lower()\n# X['description_2'] = X['description_2'].str.lower()\n\n# X['title_eq'] = X[['title_1', 'title_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\n# X['descr_eq'] = X[['description_1', 'description_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\n\n# X['title_len1'] = X['title_1'].apply(lambda x: len(str(x)))\n# X['title_len2'] = X['title_2'].apply(lambda x: len(str(x)))\n# X['title_wrd1'] = X['title_1'].apply(lambda x: len(str(x).split()))\n# X['title_wrd2'] = X['title_2'].apply(lambda x: len(str(x).split()))\n\n# X['descr_len1'] = X['description_1'].apply(lambda x: len(str(x)))\n# X['descr_len2'] = X['description_2'].apply(lambda x: len(str(x)))\n# X['descr_wrd1'] = X['description_1'].apply(lambda x: len(str(x).split()))\n# X['descr_wrd2'] = X['description_2'].apply(lambda x: len(str(x).split()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n!pip install pymorphy2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport re\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom xgboost import XGBClassifier\n\nimport pymorphy2\nfrom nltk.tokenize import word_tokenize\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/Category.csv.zip')\npairs_train = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/ItemPairs_train.csv.zip')\nitem_info_train = pd.read_csv('/kaggle/input/avito-duplicate-ads-detection/ItemInfo_train.csv.zip')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Анализ текста","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nimport re\nimport string\nmorph = pymorphy2.MorphAnalyzer()\n\ndef del_stop_words(tokens):\n    filtered_tokens = []\n    for token in tokens:\n        if token not in stopwords.words(\"russian\"):\n            filtered_tokens.append(token)\n            \n    filtered_tokens = [x for x in filtered_tokens if not re.fullmatch('[' + string.punctuation + ']+', x)]\n    return filtered_tokens\n\ndef tokenize(text):\n    if not text:\n        print('The text to be tokenized is a None type. Defaulting to blank string.')\n        text = ''\n    return word_tokenize(text, 'russian')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pairs_train.merge(item_info_train[['itemID', 'title', 'description']], how='inner', left_on='itemID_1', right_on='itemID')\nX = X.rename(columns={'title':'title_1', 'description':'description_1'})\nX = X.merge(item_info_train[['itemID', 'title', 'description']], how ='inner', left_on='itemID_2', right_on='itemID')\nX = X.rename(columns={'title': 'title_2', 'description':'description_2'})\nX = X[['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate']]\nX = X.dropna()\nX_t = X[:100000]\n\n# токенизация\nX_t['title_1'] = X_t['title_1'].apply(tokenize)\nX_t['title_2'] = X_t['title_2'].apply(tokenize)\nX_t['description_1'] = X_t['description_1'].apply(tokenize)\nX_t['description_2'] = X_t['description_2'].apply(tokenize)\n\n# удаление стоп слов\nX_t['title_1'] = X_t['title_1'].apply(del_stop_words)\nX_t['title_2'] = X_t['title_2'].apply(del_stop_words)\nX_t['description_1'] = X_t['description_1'].apply(del_stop_words)\nX_t['description_2'] = X_t['description_2'].apply(del_stop_words)\n\n# леммитизация\nX_t['title_1'] = X_t['title_1'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['title_2'] = X_t['title_2'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['description_1'] = X_t['description_1'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\nX_t['description_2'] = X_t['description_2'].apply(lambda x: [morph.parse(word)[0].normal_form for word in x])\n\n# фичи\nX_t['title_eq'] = X_t[['title_1', 'title_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\nX_t['descr_eq'] = X_t[['description_1', 'description_2']].apply(lambda x: 1 if x[0] == x[1] else 0, axis=1)\n\nX_t['title_len1'] = X_t['title_1'].apply(lambda x: len(str(x)))\nX_t['title_len2'] = X_t['title_2'].apply(lambda x: len(str(x)))\nX_t['title_wrd1'] = X_t['title_1'].apply(lambda x: len(str(x).split()))\nX_t['title_wrd2'] = X_t['title_2'].apply(lambda x: len(str(x).split()))\n\nX_t['descr_len1'] = X_t['description_1'].apply(lambda x: len(str(x)))\nX_t['descr_len2'] = X_t['description_2'].apply(lambda x: len(str(x)))\nX_t['descr_wrd1'] = X_t['description_1'].apply(lambda x: len(str(x).split()))\nX_t['descr_wrd2'] = X_t['description_2'].apply(lambda x: len(str(x).split()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = X_t.drop(['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate'], axis=1)\ny = X_t['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\nxgb = XGBClassifier()\nxgb.fit(X_train, y_train)\n\ny_test_predict = xgb.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Хэши","metadata":{}},{"cell_type":"code","source":"def hamming_distance(s1, s2):\n    return sum(ch1 != ch2 for ch1,ch2 in zip(s1,s2))\n\ndef dist(image_hash_1, image_hash_2):\n    dists = []\n    for h1 in image_hash_1:\n        for h2 in image_hash_2:\n            dists.append(hamming_distance(h1, h2))\n    \n    return min(dists)\n\ndef get_hash(s):\n    hashes_list = []\n    for id in s:\n        #if id not in skip:\n        hashes_list.append(hashes.loc[id]['image_hash'])\n        \n    return hashes_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashes = pd.DataFrame()\nfor n in range(10):\n    hash_n = pd.read_csv('/kaggle/input/hashes/image_hash_' + str(n) + '.csv')\n    hash_n['image_id'] = hash_n['image_id'].apply(lambda x: re.sub('.+/\\d+/','',x))\n    hashes = hashes.append(hash_n)\n    \nhashes = hashes.drop(['Unnamed: 0'], axis=1)\nhashes.index = hashes.image_id\nhashes = hashes[['image_hash']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_h = pairs_train.merge(item_info_train[['itemID', 'images_array']], how='inner', left_on='itemID_1', right_on='itemID')\nX_h = X_h.rename(columns={'title':'title_1', 'images_array':'images_array_1'})\nX_h = X_h.merge(item_info_train[['itemID', 'images_array']], how ='inner', left_on='itemID_2', right_on='itemID')\nX_h = X_h.rename(columns={'title': 'title_2', 'images_array':'images_array_2'})\nX_h = X_h[['images_array_1', 'images_array_2', 'isDuplicate']]\nX_h = X_h.dropna()\nX_h = X_h[:100000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_img_ids = []\n# for i in X_h.images_array_1:\n#     all_img_ids += i\n    \n# for i in X_h.images_array_2:\n#     all_img_ids += i\n\n# skip = []\n# for ind in all_img_ids:\n#     try:\n#         hashes.loc[ind]\n#     except:\n#         skip.append(ind)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_h['images_array_1'] = X_h['images_array_1'].apply(lambda x: str(x).split(', '))\nX_h['images_array_2'] = X_h['images_array_2'].apply(lambda x: str(x).split(', '))\n\nX_h['image_hash_1'] = X_h['images_array_1'].apply(get_hash)\nX_h['image_hash_2'] = X_h['images_array_2'].apply(get_hash)\n\nX_h['dist'] = X_h[['image_hash_1', 'image_hash_2']].apply(lambda x: dist(x['image_hash_1'], x['image_hash_2']), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = X_h[['dist']]\ny = X_h['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\nxgb = XGBClassifier()\nxgb.fit(X_train, y_train)\n\ny_test_predict = xgb.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Анализ текста + Хэши","metadata":{}},{"cell_type":"code","source":"X_t['dist'] = X_h['dist']\nX_t = X_t.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = X_t.drop(['title_1', 'title_2', 'description_1', 'description_2', 'isDuplicate'], axis=1)\ny = X_t['isDuplicate']\nX_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.2)\n\nxgb = XGBClassifier()\nxgb.fit(X_train, y_train)\n\ny_test_predict = xgb.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, y_test_predict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}