{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "87d6ca82-f28d-d73f-d1c5-2c61ab10f103" }, "source": [ "The purpose of this doc is to explore possible ways of solving this classification problem. \n", "\n", "Given that training data on physics topics is unknown, we can not use traditional supervised learning, which lands us into three possible directions:\n", "\n", " 1. Unsupervised learning - Such clustering techniques\n", " 2. Rule-based algorithms - which I took a stab at but the result isn't very impressive\n", " **3. psudo-supervised learning through data transformation**\n", "\n", "In this doc I'd like to tinker with the third option. **I'm aiming to build a pool of topic-specific vocabulary pool which is going to become the source of tags.** \n", "\n", "\n", "**(TBD)**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a2db11e8-b9ba-530f-25f2-a6cb9e921c65" }, "outputs": [], "source": [ "## This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import nltk # natural language processing\n", "import re # regular expression\n", "from bs4 import BeautifulSoup #scraping HTML\n", "from nltk.corpus import stopwords\n", "import seaborn as sns # visualization\n", "import matplotlib.pyplot as plt\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.feature_extraction.text import TfidfTransformer\n", "from string import punctuation\n", "from nltk.collocations import BigramCollocationFinder\n", "\n", "\n", "\n", "# nltk workspace\n", "\n", "stop = set(stopwords.words('english'))\n", "lemmatizer = nltk.stem.WordNetLemmatizer()\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fa1fb9df-76f3-8761-8986-49b9118402dc" }, "outputs": [], "source": [ "# Define function\n", "def strip_punctuation(s):\n", " # input str, output str, strip out punctuations\n", " return ''.join(c for c in s if c not in punctuation)\n", "\n", "def remove_html(s):\n", " #input str, output str, remove html from content\n", " soup = BeautifulSoup(s,'html.parser')\n", " content = soup.get_text()\n", " return content\n", "\n", "def text_transform(dataframe):\n", " # input data frame, process title and content. \n", " dataframe['title'] = dataframe['title'].apply(lambda x: strip_punctuation(str.lower(x)))\n", " dataframe['content'] = dataframe['content'].apply(lambda x: strip_punctuation(str.lower(remove_html(x).replace(\"\\n\",\" \"))))\n", "\n", "def load_data(name):\n", " utl = \"../input/\"+name+'.csv'\n", " files = pd.read_csv(utl)\n", " text_transform(files)\n", " files['category'] = name\n", " return files\n", "\n", "def merge_data(list_of_files):\n", " list_of_dataframe = [\"\"]*len(list_of_files)\n", " for i in range(0,len(list_of_files)):\n", " list_of_dataframe[i] = load_data(list_of_files[i])\n", " data = pd.concat(list_of_dataframe,axis = 0, ignore_index =True)\n", " return data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3b75d47e-75c0-25a2-b0a8-ceab1a51d55c" }, "outputs": [], "source": [ "def list_to_str(lists):\n", " # input list, output string.\n", " strs = \"\"\n", " for content in lists:\n", " strs+=content\n", " return strs\n", "\n", "def to_plain_text(dataframe):\n", " # input dataframe, output str. transform the text column in dataframe to str\n", " text=list_to_str(dataframe['all_text'].apply(lambda x: x.replace('\\n',' ')).tolist())\n", " return text\n", "\n", "def to_nltk_text(dataframe):\n", " #input dataframe, output nltk text object. \n", " text = to_plain_text(dataframe)\n", " token = nltk.word_tokenize(text)\n", " lemmatizer = nltk.stem.WordNetLemmatizer()\n", " lemmas = [lemmatizer.lemmatize(t) for t in token]\n", " return nltk.Text(lemmas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ccd7171c-2094-9406-4174-19ecf4b554f2" }, "outputs": [], "source": [ "def hasNumbers(inputString):\n", " return any(char.isdigit() for char in inputString)\n", "\n", "\n", "def freqDist(text,include_bigram = True):\n", " #input str, output dictionary of word and count pair. Calculate (absolute) term frequencies.\n", " #Incorporate bigrams into the model\n", " text_bigr = []\n", " if include_bigram == True:\n", " text_bigr = list(nltk.bigrams(all_text))\n", " freqDist = {}\n", " for data in [text,text_bigr]:\n", " for word in data:\n", " if word in freqDist:\n", " freqDist[word] +=1\n", " else:\n", " freqDist[word] = 1\n", " return freqDist\n", "\n", "def relativeFreq(subset,alls,sort=True,adjusted=0):\n", " #input subset and alls are dictionaries from freqDist function. subset is a subset of text from \n", " #the specific topic we are interested in studying whereas alls the totality of text data. we have\n", " #at disposal.if sort equals to True, output will be sorted based on relative frequencies. Adjusted \n", " #is a manual adjustment to terms that have an overall low volume.\n", " result = [\" \"]*len(subset)\n", " result_dict = {}\n", " modifier = 1\n", " for i, key in enumerate(subset.keys()):\n", " if alls[key] > adjusted and hasNumbers(key) == False:\n", " modifier = 1\n", " else:\n", " modifier = 0\n", " tf = float(subset[key])/alls[key]\n", " result[i]=(key,tf*modifier)\n", " result_dict[key] = tf*modifier\n", " if sort == True:\n", " result.sort(key=lambda tup: tup[1], reverse = True)\n", " return [result,result_dict]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c2763094-4f0c-29b4-8ad5-bd43cdbe89fc" }, "outputs": [], "source": [ "#Taking biology as an example\n", "list_of_files = ['biology','cooking','crypto','diy','robotics','travel','test']\n", "data = merge_data(list_of_files)\n", "data['all_text'] = data['title'] + \" \" + data['content']\n", "all_text = to_nltk_text(data)\n", "Fdist_all = freqDist(all_text)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0fea6db6-e956-e8eb-ae32-287e695c4834" }, "outputs": [], "source": [ "#Define Function tag Explained\n", "def tagExplained(s,all_text,Fdist_all):\n", " #s, string, category of interest.\n", " interest = to_nltk_text(data[data['category'] == s])\n", " Fdist_interest = freqDist(interest)\n", " relative_Freq_dict = relativeFreq(Fdist_interest,Fdist_all)[1]\n", " tags=data[data['category']==s]['tags'].apply(lambda x:nltk.word_tokenize(x)).tolist()\n", " tags = [x for record in tags for x in record]\n", " tags=[(lemmatizer.lemmatize(x.split('-')[0]),lemmatizer.lemmatize(x.split('-')[1])) if \"-\" in x else lemmatizer.lemmatize(x) for x in tags]\n", " relative_score = [relative_Freq_dict[x] if x in relative_Freq_dict else -1.0 for x in tags ]\n", " per_of_tag_explained = sum(1 if x != -1.0 else 0 for x in relative_score)/float(len(relative_score))\n", " return relative_score,per_of_tag_explained\n", "\n", " \n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b386164f-ddee-314e-995d-eaab4997ac7d" }, "outputs": [], "source": [ "#f,axes = plt.subplots(1,1,figsize=(10,10),sharex=False,sharey=False)\n", "tag_explained = [0.0]*len(list_of_files[:-1])\n", "score = [\"\"]*len(list_of_files[:-1])\n", "for i,category in enumerate(list_of_files[:-1]):\n", " score[i],tag_explained[i] = tagExplained(category,all_text,Fdist_all)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dc77ae57-3300-2795-b876-7ec6d5609663" }, "outputs": [], "source": [ "for i in range(0,len(list_of_files[:-1])):\n", " sns.distplot(score[i],label = list_of_files[i],hist=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d5b87b67-bde9-6025-46ca-2bc4b03a9c48" }, "outputs": [], "source": [ "#Tag prediction for test dataset\n", "test = data[data['category'] == 'test']\n", "vocabulary_text = to_nltk_text(test)\n", "Fdist_test = freqDist(vocabulary_text)\n", "relative_freq, relative_freq_dict = relativeFreq(Fdist_test,Fdist_all)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d3ee4bd0-cf67-9c07-c284-6a29c4e82eb0" }, "outputs": [], "source": [ "test['all_text_lemma'] = test['all_text'].apply(lambda x: [lemmatizer.lemmatize(token) for token in nltk.word_tokenize(x.replace('\\n',' '))])\n", "test['all_text_lemma2'] = test['all_text_lemma'].apply(lambda x: x+list(nltk.bigrams(x)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d76b9aa5-fbd3-c110-df20-00b6d949db12" }, "outputs": [], "source": [ "import heapq\n", "Top_N = 5\n", "#top_Per = sum([sum(x)/len(x) for x in score])/len(score)\n", "top_Per =0.9\n", "print(top_Per)\n", "def pickTheBest(l):\n", " result = {}\n", " for lemma in l:\n", " if lemma in relative_freq_dict:\n", " result[relative_freq_dict[lemma]] = lemma\n", " return result\n", "test['relative_freq'] = test['all_text_lemma2'].apply(pickTheBest)\n", "\n", "def tags(dic):\n", " result = heapq.nlargest(Top_N,list(dic.keys()))\n", " result = [dic[x] for x in result if x>=top_Per]\n", " result = \"\".join([x[0]+\"-\"+x[1]+\" \"if type(x) == tuple else x+\" \" for x in result])\n", " return result\n", " \n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b02bc69f-9967-52c1-a9be-f30be428642a" }, "outputs": [], "source": [ "test['tags'] = test['relative_freq'].apply(tags)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "50adc703-6e45-90eb-d4db-4f40f1c32b98" }, "outputs": [], "source": [ "test[['all_text','tags']][0:100]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "443c55bf-8871-6bfc-ff4b-15d82fdf7c9c" }, "outputs": [], "source": [ "test[['id','tags']].to_csv('submission.csv',index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7d30c0cb-3938-fe4f-6fe2-968a47a750f4" }, "outputs": [], "source": [ "# Visualization: How many of the tags are included in the category vocabulary?\n", "#sns.barplot(x=list_of_files[:-1],y=tag_explained,color=sns.light_palette((210, 90, 60), input=\"husl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "573b10a3-3483-b9fa-87b1-164a6e4ab2a1" }, "outputs": [], "source": [ "##all_text = to_nltk_text(data)\n", "#biology = to_nltk_text(data[data['category'] == 'biology'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f53b7150-025a-3638-8055-b39e6f3663ec" }, "outputs": [], "source": [ "#Fdist_all = freqDist(all_text)\n", "#Fdist_biology = freqDist(biology)\n", "#relative_Freq,relative_Freq_dict = relativeFreq(Fdist_biology,Fdist_all)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "fd163bf7-ae8e-da4d-c38c-24c2d6e47a7d" }, "source": [ "It actually looks descent and includes quite a bit of topic-specific terms for biology.\n", "\n", "\n", "## **Next Steps** ##\n", "\n", "- Refine The process: Possibly could clean the data better. (for instance plurals..numbers,.etc)\n", "- Ngrams? \n", "- Study the tags" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1b5ad051-6d16-c64e-995e-27f6d94d0dd3" }, "outputs": [], "source": [ "#\n", "#tags=data[data['category']=='biology']['tags'].apply(lambda x: [lemmatizer.lemmatize(t) for t in nltk.word_tokenize(x)]).tolist()\n", "#tags=[tag for record in tags for tag in record]\n", "#freqTag = freqDist(tags,include_bigram = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8100b426-02c4-f0c1-642a-5f18e34e6a00" }, "outputs": [], "source": [ "##freqtag_df = pd.DataFrame.from_dict(freqTag,orient='index')\\\n", " #.reset_index()\\\n", " #.rename(columns={'index':'tags',0:\"freq\"})\\\n", " #.sort_values('freq',axis=0,ascending=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "340be7e6-8c0f-6049-e84f-9472b74fafba" }, "outputs": [], "source": [ "##freqtag_df['tag_revised'] = freqtag_df['tags'].apply(lambda x: (x.split('-')[0],x.split('-')[1]) if \"-\" in x else x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "07c98f69-59f8-f01e-9585-e97f600aa3bb" }, "outputs": [], "source": [ "##freqtag_df[0:100]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "298346f1-84ff-057f-2005-5cf0c08dc2d2" }, "outputs": [], "source": [ "##freqtag_df['freq_p'] = freqtag_df['freq'].apply(lambda x: float(x)/freqtag_df.freq.sum())\n", "#freqtag_df['relative_score'] = freqtag_df['tag_revised'].apply(lambda x: relative_Freq_dict[x] if x in relative_Freq_dict else -1.0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8d67787a-d2d1-d5f2-0a55-b184971ee2cc" }, "outputs": [], "source": [ "#investigation = freqtag_df[freqtag_df['relative_score']==-1.0]\n", "#print(len(investigation))\n", "#investigation[0:50]\n", "# The majority of unidentifiable seems to come from compounded word - maybe we should also consider bigram " ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }