{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "49bdf4a7-2285-15ca-7970-8a353ed8ebae" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ba94f9bc-0862-d7b7-c9ac-e899eca1e70f" }, "outputs": [], "source": [ "import re\n", "import csv\n", "import operator \n", "from collections import defaultdict\n", "\n", "stop_words = set(['a', \"a's\", 'able', 'about', 'above', 'according', 'accordingly', 'across', 'actually', 'after', 'actual','afterwards', 'again', 'against', \"ain't\", 'all', 'allow', 'allows', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'an', 'and', 'another', 'any', 'anybody', 'anyhow', 'anyone', 'anything', 'anyway', 'anyways', 'anywhere', 'apart', 'appear', 'appreciate', 'appropriate', 'are', \"aren't\", 'around', 'as', 'aside', 'ask', 'asking', 'associated', 'at', 'available', 'away', 'awfully', 'b', 'be', 'became', 'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'believe', 'below', 'beside', 'besides', 'best', 'better', 'between', 'beyond', 'both', 'brief', 'but', 'by', 'c', \"c'mon\", \"c's\", 'came', 'can', \"can't\", 'cannot', 'cant', 'cause', 'causes', 'certain', 'certainly', 'changes', 'clearly', 'co', 'com', 'come', 'comes', 'concerning', 'consequently', 'consider', 'considering', 'contain', 'containing', 'contains', 'corresponding', 'could', \"couldn't\", 'course', 'currently', 'd', 'definitely', 'described', 'despite', 'did', \"didn't\", 'different', 'do', 'does', \"doesn't\", 'doing', \"don't\", 'done', 'down', 'downwards', 'during', 'e', 'each', 'edu', 'eg', 'eight', 'either', 'else', 'elsewhere', 'enough', 'entirely', 'especially', 'et', 'etc', 'even', 'ever', 'every', 'everybody', 'everyone', 'everything', 'everywhere', 'ex', 'exactly', 'example', 'except', 'f', 'far', 'few', 'fifth', 'first', 'five', 'followed', 'following', 'follows', 'for', 'former', 'formerly', 'forth', 'four', 'from', 'further', 'furthermore', 'g', 'get', 'gets', 'getting', 'given', 'gives', 'go', 'goes', 'going', 'gone', 'got', 'gotten', 'greetings', 'h', 'had', \"hadn't\", 'happens', 'hardly', 'has', \"hasn't\", 'have', \"haven't\", 'having', 'he', \"he's\", 'hello', 'help', 'hence', 'her', 'here', \"here's\", 'hereafter', 'hereby', 'herein', 'hereupon', 'hers', 'herself', 'hi', 'him', 'himself', 'his', 'hither', 'hopefully', 'how', 'howbeit', 'however', 'i', \"i'd\", \"i'll\", \"i'm\", \"i've\", 'ie', 'if', 'ignored', 'immediate', 'in', 'inasmuch', 'inc', 'indeed', 'indicate', 'indicated', 'indicates', 'inner', 'insofar', 'instead', 'into', 'inward', 'is', \"isn't\", 'it', \"it'd\", \"it'll\", \"it's\", 'its', 'itself', 'j', 'just', 'k', 'keep', 'keeps', 'kept', 'know', 'knows', 'known', 'l', 'last', 'lately', 'later', 'latter', 'latterly', 'least', 'less', 'lest', 'let', \"let's\", 'like', 'liked', 'likely', 'little', 'look', 'looking', 'looks', 'ltd', 'm', 'mainly', 'many', 'may', 'maybe', 'me', 'mean', 'meanwhile', 'merely', 'might', 'more', 'moreover', 'most', 'mostly', 'much', 'must', 'my', 'myself', 'n', 'name', 'namely', 'nd', 'near', 'nearly', 'necessary', 'need', 'needs', 'neither', 'never', 'nevertheless', 'new', 'next', 'nine', 'no', 'nobody', 'non', 'none', 'noone', 'nor', 'normally', 'not', 'nothing', 'novel', 'now', 'nowhere', 'o', 'obviously', 'of', 'off', 'often', 'oh', 'ok', 'okay', 'old', 'on', 'once', 'one', 'ones', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'ought', 'our', 'ours', 'ourselves', 'out', 'outside', 'over', 'overall', 'own', 'p', 'particular', 'particularly', 'per', 'perhaps', 'placed', 'please', 'plus', 'possible', 'presumably', 'probably', 'provides', 'q', 'que', 'quite', 'qv', 'r', 'rather', 'rd', 're', 'really', 'reasonably', 'regarding', 'regardless', 'regards', 'relatively', 'respectively', 'right', 's', 'said', 'same', 'saw', 'say', 'saying', 'says', 'second', 'secondly', 'see', 'seeing', 'seem', 'seemed', 'seeming', 'seems', 'seen', 'self', 'selves', 'sensible', 'sent', 'serious', 'seriously', 'seven', 'several', 'shall', 'she', 'should', \"shouldn't\", 'since', 'six', 'so', 'some', 'somebody', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhat', 'somewhere', 'soon', 'sorry', 'specified', 'specify', 'specifying', 'still', 'sub', 'such', 'sup', 'sure', 't', \"t's\", 'take', 'taken', 'tell', 'tends', 'th', 'than', 'thank', 'thanks', 'thanx', 'that', \"that's\", 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'thence', 'there', \"there's\", 'thereafter', 'thereby', 'therefore', 'therein', 'theres', 'thereupon', 'these', 'they', \"they'd\", \"they'll\", \"they're\", \"they've\", 'think', 'third', 'this', 'thorough', 'thoroughly', 'those', 'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together', 'too', 'took', 'toward', 'towards', 'tried', 'tries', 'truly', 'try', 'trying', 'twice', 'two', 'u', 'un', 'under', 'unfortunately', 'unless', 'unlikely', 'until', 'unto', 'up', 'upon', 'us', 'use', 'used', 'useful', 'uses', 'using', 'usually', 'uucp', 'v', 'value', 'various', 'very', 'via', 'viz', 'vs', 'w', 'want', 'wants', 'was', \"wasn't\", 'way', 'we', \"we'd\", \"we'll\", \"we're\", \"we've\", 'welcome', 'well', 'went', 'were', \"weren't\", 'what', \"what's\", 'whatever', 'when', 'whence', 'whenever', 'where', \"where's\", 'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', \"who's\", 'whoever', 'whole', 'whom', 'whose', 'why', 'will', 'willing', 'wish', 'with', 'within', 'without', \"won't\", 'wonder', 'would', 'would', \"wouldn't\", 'x', 'y', 'yes', 'yet', 'you', \"you'd\", \"you'll\", \"you're\", \"you've\", 'your', 'yours', 'yourself', 'yourselves', 'z', 'zero', ''])\n", "\n", "\n", "def f1score(tp,fp,fn):\n", " p=(tp*1.)/(tp+fp)\n", " r=(tp*1.)/(tp+fn)\n", " f1=(2*p*r)/(p+r)\n", " return f1\n", "def cleantext(raw_html):\n", " cleanr=re.compile('<.*?>')\n", " #check this out\n", " cleantext=re.sub(cleanr,'',raw_html)\n", " return cleantext\n", "def get_words(text):\n", " word_split=re.compile('[^a-zA-Z0-9_\\\\+\\\\-/]') #check out regX\n", " return [word.strip().lower() for word in word_split.split(text)]\n", "datapath=\"../input\"\n", "in_file=open(datapath+\"/test.csv\")\n", "out_file=open(\"sub_freq.csv\",\"w\")\n", "reader=csv.DictReader(in_file)\n", "writer=csv.writer(out_file)\n", "writer.writerow(['id','tags'])\n", "for ind,row in enumerate(reader):\n", " text=cleantext(row['title'])\n", " tfrequency_dict=defaultdict(int)\n", " word_count=0.\n", " for word in get_words(text):\n", " if word not in stop_words and word.isalpha():\n", " tfrequency_dict[word]+=1\n", " word_count+=1.\n", " for word in tfrequency_dict:\n", " tf=tfrequency_dict[word]/word_count\n", " tfrequency_dict[word]=tf\n", " pred_title_tags=sorted(tfrequency_dict,key=tfrequency_dict.get,reverse=True)[:10]\n", " text=cleantext(row['content'])\n", " dfrequency_dict=defaultdict(int)\n", " word_count=0.\n", " for word in get_words(text):\n", " if word not in stop_words and word.isalpha():\n", " dfrequency_dict[word]+=1\n", " word_count+=1.\n", " for word in dfrequency_dict:\n", " tf=dfrequency_dict[word]/word_count\n", " dfrequency_dict[word]=tf\n", " pred_content_tags=sorted(dfrequency_dict,key=dfrequency_dict.get,reverse=True)[:10]\n", " pred_tags_dict={}\n", " for word in set(pred_title_tags+pred_content_tags):\n", " pred_tags_dict[word]=tfrequency_dict.get(word,0)+dfrequency_dict.get(word,0)\n", " pred_tags=set(sorted(pred_tags_dict,key=pred_tags_dict.get,reverse=True)[:3])\n", " writer.writerow([row['id'],\" \".join(pred_tags)])\n", " if ind%50000==0:\n", " print(\"processed\",ind)\n", "in_file.close()\n", "out_file.close()\n", " \n", "\n", "\n", "\n", "\n", "\n", "\n" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }