{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "2567d431-4758-be7d-20c8-cf4367cf9416" }, "source": [ "Text Preprocessing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f06b2102-715e-bbe3-1d6a-e908b8a9cfab" }, "outputs": [], "source": [ "import pandas as pd\n", "from bs4 import BeautifulSoup\n", "from nltk.corpus import stopwords\n", "import re\n", "import string\n", "\n", "import numpy as np\n", "import seaborn as sns\n", "import itertools \n", "import csv\n", "import collections\n", "import matplotlib.pyplot as plt\n", "\n", "sns.set_context(\"paper\")\n", "%matplotlib inline\n", "\n", "RES_DIR = \"../input/\"" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f5a26cd8-f817-ae12-4e01-1d66e149f276" }, "source": [ "Datasets loading\n", "---------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "048006aa-02db-a66b-3394-48f4832ed4a9" }, "outputs": [], "source": [ "# Load train data (skips the content column)\n", "def load_train_data():\n", " categories = ['cooking', 'robotics', 'travel', 'crypto', 'diy', 'biology']\n", " train_data = []\n", " for cat in categories:\n", " data = pd.read_csv(\"{}{}.csv\".format(RES_DIR, cat), usecols=['id', 'title', 'tags'])\n", " data['category'] = cat\n", " train_data.append(data)\n", " \n", " return pd.concat(train_data)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25747165-b9d8-ed21-c677-0366c516728f" }, "outputs": [], "source": [ "train_data = load_train_data()\n", "#import the test data\n", "test = pd.read_csv(\"../input/test.csv\")\n", "train_data.head()\n", "test.head()\n", "\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.feature_extraction import DictVectorizer\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.decomposition import NMF, LatentDirichletAllocation\n", "\n", "convec = CountVectorizer(max_df=0.95, min_df=2,stop_words='english')\n", "corpus = test['content'].values\n", "ldavec = LatentDirichletAllocation( max_iter=5,learning_method='online',learning_offset=50.,random_state=0)\n", "Xtf = convec.fit_transform(corpus)\n", "Ytf = ldavec.fit(Xtf)\n", "tf_names = Xtf.get_feature_names()\n", "print(tf_names)\n" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "99969408-bcb7-eff1-cf9a-2ca439874b23" }, "source": [ "Removing html tags and uris from contents\n", "-----------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4e8b3030-304e-4033-0607-3007af3bdfaa" }, "outputs": [], "source": [ "uri_re = r'(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:\\'\".,<>?\u00ab\u00bb\u201c\u201d\u2018\u2019]))'\n", "\n", "def stripTagsAndUris(x):\n", " if x:\n", " # BeautifulSoup on content\n", " soup = BeautifulSoup(x, \"html.parser\")\n", " # Stripping all tags with their content if any\n", " if soup.code:\n", " soup.code.decompose()\n", " # Get all the text out of the html\n", " text = soup.get_text()\n", " # Returning text stripping out all uris\n", " return re.sub(uri_re, \"\", text)\n", " else:\n", " return \"\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b79c4012-de24-3437-752d-cfd54380c0f1" }, "outputs": [], "source": [ "# This could take a while\n", "train_data[\"title\"] = train_data[\"title\"].map(stripTagsAndUris)\n", "test[\"content\"] = test[\"content\"].map(stripTagsAndUris)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b8b24598-28db-2fe7-5cf4-3ca79703bdf0" }, "outputs": [], "source": [ "train_data.head()\n", "test.head()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c3c14702-c42b-8078-fb5d-8551bfb5022c" }, "source": [ "Removing punctuation from titles and contents\n", "-----------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f79fc221-3ef7-9d15-31a2-9453af26986b" }, "outputs": [], "source": [ "def removePunctuation(x):\n", " # Lowercasing all words\n", " x = x.lower()\n", " # Removing non ASCII chars\n", " x = re.sub(r'[^\\x00-\\x7f]',r' ',x)\n", " # Removing (replacing with empty spaces actually) all the punctuations\n", " return re.sub(\"[\"+string.punctuation+\"]\", \" \", x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "21967d22-6f02-11fd-15b8-14b3947a63f2" }, "outputs": [], "source": [ "train_data[\"title\"] = train_data[\"title\"].map(removePunctuation)\n", "test[\"title\"] = test[\"title\"].map(removePunctuation)\n", "test[\"content\"] = test[\"content\"].map(removePunctuation)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ff6788cc-6747-546c-a8a4-76d6dc691084" }, "outputs": [], "source": [ "train_data.head()\n", "test.head()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "1c81c133-dff7-d567-9d2b-c619fdfb7bd4" }, "source": [ "Removing stopwords from titles and contents\n", "-----------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d015616b-e2ca-4318-90ef-d0357659a9e8" }, "outputs": [], "source": [ "stops = set(stopwords.words(\"english\"))\n", "def removeStopwords(x):\n", " # Removing all the stopwords\n", " filtered_words = [word for word in x.split() if word not in stops]\n", " return \" \".join(filtered_words)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f3817c69-1e8a-2f87-5b4c-5e8fe6c88074" }, "outputs": [], "source": [ " train_data[\"title\"] = train_data[\"title\"].map(removeStopwords)\n", " test[\"title\"] = test[\"title\"].map(removeStopwords)\n", " test[\"content\"] = test[\"content\"].map(removeStopwords)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "508188ff-4f19-18e7-c4a1-1a6936973496" }, "outputs": [], "source": [ "test.head()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "87a4094d-91e2-ad64-35bb-1b78e4e9f57d" }, "source": [ "Splitting tags string in a list of tags\n", "-----------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c88d201b-10a5-acb9-4ee3-c0d3155742bb" }, "outputs": [], "source": [ "# Summary about tags\n", "tag_lists = [t.split() for t in train_data['tags'].values]\n", "tag_lists2 = [t.split() for t in train_data['title'].values]\n", "all_tags = list(itertools.chain(*tag_lists,*tag_lists2))\n", "tag_list_size = np.array([len(x) for x in tag_lists])\n", "print(\"\"\"The corpus is composed by {} questions. Overall {} tags have been used, of which {} unique ones. \n", "Average number of tags per question {:.2f} (min={}, max={}, std={:.2f})\"\"\".format(\n", " len(train_data),\n", " len(all_tags), len(set(all_tags)),\n", " tag_list_size.mean(), \n", " min(tag_list_size), max(tag_list_size),\n", " tag_list_size.std()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80914ebf-8106-6b09-0f20-f4d01488e225" }, "outputs": [], "source": [ "# Utility function to return top occuring tags in the passed df\n", "def get_top_tags(df, n=None):\n", " itag_lists = [t.split() for t in df['tags'].values]\n", " itag_lists2 = [t.split() for t in df['title'].values]\n", " tags = list(itertools.chain(*itag_lists,*itag_lists2))\n", " top_tags = collections.Counter(list(tags)).most_common(n)\n", " tags, count = zip(*top_tags)\n", " return tags, count\n", "# Utility function to return top occuring tags in the passed df" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a53474ac-bb82-b921-2e04-58dbe6e62289" }, "outputs": [], "source": [ "# Created DataFrame indexed on tags\n", "tags_df = pd.DataFrame(index=set(itertools.chain(*tag_lists,*tag_lists2)))\n", "# For each category create a column and update the flag to tag count\n", "for i, (name, group) in enumerate(train_data.groupby('category')):\n", " tags_df[name] = 0\n", " tmp_index, count = get_top_tags(group)\n", " tmp = pd.Series(count, index=tmp_index)\n", " tags_df[name].update(tmp)\n", "# Number of categories for which a tag appeared at least 1 time\n", "tags_df['categories_appears'] = tags_df.apply(lambda x: x.astype(bool).sum(), axis=1)\n", "tags_df['categories_appears'].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b6428ee4-80b4-52ad-55c0-921ff13a90cb" }, "outputs": [], "source": [ "# viewing the table of tags\n", "from sklearn import preprocessing\n", "A=tags_df\n", "del A['categories_appears']\n", "A_n = preprocessing.normalize(A, norm='l2')\n", "print(A_n)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0e713c77-ea7c-815f-dd9f-a66f0a51586c" }, "source": [ "#Solving the question with a Singular Value Decomposition, \n", "#this is the core function\n", "-----------" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c1291508-262d-13ea-400e-4afe585e9b4a" }, "outputs": [], "source": [ "from numpy.linalg import inv\n", "U,s,V=np.linalg.svd(A,full_matrices=False)\n", "# reconstruct\n", "S=np.diag(s)\n", "\n", "iS=inv(S)\n", "US=np.dot(U,iS)\n", "US\n", "# A fill up with US matrix\n", "US_df=pd.DataFrame(data=US, index=tags_df.index, columns=tags_df.columns)\n", "# with this simple math i know all the relations between all the tags and the documents\n", "# " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8091b509-9ad6-9fda-b98a-75cfa3bab9ca" }, "outputs": [], "source": [ "#learn how to use dataframes... and yes the algorithm knows extreme tourism antarctica has something to do with travel...\n", "df1=US_df['extreme-tourism':'extreme-tourism':]\n", "df2=US_df['antarctica':'antarctica':]\n", "frames = [df1,df2]\n", "Qtemp=pd.concat(frames).sum()\n", "np.dot(Qtemp,V)/np.dot(np.abs(Qtemp),np.abs(V))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9b38d8f1-c8dd-1e4d-b726-844a4899bdd8" }, "source": [ "the training tells me its 100% travel, and 20% biology wtf ?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "10f36d93-e785-9d08-65be-030875959124" }, "outputs": [], "source": [ "def taggifytitle(x):\n", " tempspl = x.strip().split() \n", " Qtemp=newDF\n", " for sword in tempspl:\n", " if sword in US_df.index:\n", " #print(US_df.loc[sword:sword,:])\n", " Qtemp=Qtemp.append(US_df.loc[sword:sword,:])\n", " #print(Qtemp)\n", " simila=np.dot(Qtemp.sum(),V)/np.dot(np.abs(Qtemp.sum()),np.abs(V))\n", " tempprnt=''\n", " for xyb in range(0,5):\n", " if simila[xyb]>0.89 or simila[xyb]==np.amax(simila[0:5]):\n", " tempprnt+=columns[xyb]+' '\n", " \n", " return tempprnt " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1316f762-0ecc-9a84-1ae1-50719385d7b1" }, "outputs": [], "source": [ "columns = ['biology','cooking','crypto','diy','robotics','travel']\n", "#,'categories_appears']\n", "data = {'biology': [0],'cooking': [0],'crypto': [0],'diy': [0],'robotics': [0],'travel': [0],'categories_appears': [0]}\n", "newDF = pd.DataFrame(data, columns=columns,index = ['blanco'])\n", "#print(newDF)\n", "stukjetesten=test['content'][0:100]\n", "stukjetesten.map(taggifytitle)\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "01e05935-735d-5dcc-51e8-73e6d10842af" }, "outputs": [], "source": [ "test.to_csv(\"test_SVDPaul.csv\", index=False)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }