{"cells":[{"metadata":{},"cell_type":"markdown","source":"![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAABj1BMVEX///////78///5///2///x/////f/0//////3t/////fz/+//q//////v5//3//vkmrOksq+Dw//uJxd7//fPl////+/fG6vItntUlrt0AZKi24+svoNEtptczmM8lrORHqNAJbbkLbbNgtdQNba8Ab8IAabD/9///9PQpquz8/+/5//jU9fnX///i+v0AXKoAWJgAXZiSw9EAXZFesMWEs8mGxdMAZbLx9//h/PDx8/D9+96nzt5eudOCushhuclxrdJ1u8H/6+zhqq7JbXjSkZe40Nqhsbfc5OQnqM0+q8XNf3/DHinXGybLaGuXwtlSg6YdWIJUiJ/539e6MzTyFSLlJTwib6ZVpdY7b4uHoa5MnrrGiYJnjKjB4vPQ8+gAY4fwxMbfppV3lcVrp7Msm+Sl3+S99vlTkb4vcZxvn7YGacdBmsKYz+Z73e8EcasmsdM2o7Y5hr+jy8t/3fsATXqFuuGqx9eq4fOp5uKNxN3+3uzaFBmvMSesyeMAS4TK0dG0Qz4YfbFHcpgATnKpTjNQAAANkElEQVR4nO2Z/XvaVpaA4eoLSegL6wMJFAQIMEiWDRhwU+x4phs3eNu1kzRLOh57u7ONv4LrTNNZe3ezM5l2//A9FzuNnbh2n0mane1z3h8CFuLqvpxzzz1SUikEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQX4J0u9vpOuGen+XuQnCSyppNIiQ4iVNZVnNfPcxP2Ib6UidbbVSvPL2p1JGuS1J0SxcNvfu17oZhudZNiUwvJwSBJZRVTWTecffN/2xyeRyXDYX8Xz09liKkmazliVY2dn38GvejMrLhPA8LxGVERTTVFmSeldFdVhdXq4uOvCrXWGoKrywcmd19c5vlswPkasqz8u//eQf7j7MqQ1F01QuXvv0niS905jcTEnX9VEBfjz1iivaUnZx1Gy66wl5p8v8XAiZ/cfPPv/sn2ZJQxLmNjbvP3CS2xn+HUZMR31fN1y/kFLYKxx427SqPhiOkiv83z8yYR5+8ejRo3++K3Fib+xVHq9++bvb9pnheRKl3yiL9O+r8+vseG5Gf9KkhpHQuPDR+XCy1rK28v6t4lbyOocvD3j+B3358Wuv5/PmtW9I9VeGn/+erI29cq2ysO3QYsBHkSS1ZAle5CjSNFtmiAKVcdbWIihHimZLvJLhZR4Wsc2wJiSkKkmmqTRklp3xSyXd35Fy3KwNCzxjq7Jsarxp25oW5Wzzo3+JHWCJO5+DpJmm1ooYVYHTmZRm2kpKsuGYysDK0WyNMKxmE2KymiIIZqTCXGAsRVVJpEamfW3G8ZCldz/7/MUXjd5XYb3sjYdZjmgKIblc1pqzxCjikr29OOYITbiMworWHJ1enGQ5NSNJRMjmckk2IikiJHPW3FyypIKhYejNgmkLXA4GsbICswSv2SVZ5pydmFOEOYuzcrMfn/38tqJyVlwoDHdiiyWZFMMKjAIHs8nOcMeJ4UJQILRMQuEUko0La07CmRorxjtDZ47jrzWESiPN/uvdP8z+23xYK7d/iAWG2FqGUXPOVhAsfp1Y/fWtoBgMxTQsWYYUVrYmRWASLB5lGU0TnIDSEdJy8vV6UF1fEW1mavjktDWb7T8GVvdfsuJhFYbb3VscFdd/I/aqweNgcSk6yzKTtTY2u/OUcNthU4wgqAqTwLF2u73w1YOhGLUUjVn56suD8f3Uvf12u3v/yw4jOpvthXb3YMjZNxgybCrDHc4/Lde8TSsyIS0yhFHUwqSkG8HeYuC6hq5PVjhGYpPhxIc/moBuFA/jyBQcONDM9zlFTrbcpvtsMaudGTZ3I/Zo3SiV/MkRp4oTP2+MvlkfGfroG+vQ1Z+UguQ8S3POD+0wrITh8XFYb/c402y1mJ3xoF6plMu14+PudnI7rQm97nG9Fg7v14ByuHB01K3XntYq4cJQuW4pwo4vCIT/88LxcSXcTCAfNAUEGYkURqUmLYmGbkDx9yexYE6tdXArFuFFLx5xZs6ZBIY+6QuaZAWlZsmtvjL0C+Jp0X327Nkfdz8STG6il5p5o5hvNieJsNos6e662DqbWXwQlmvl2mBQgxnX2xuc+XHkwPTL5Uq9Xg4rlfa2RTS2B2UC1ODcqeNxWKHv4J8D8boYpviUqvKJVzs4GHQdwdQEyEWA56lh04AA5kuGYZSMo1xLXTR0iMlywTkNqHaQmKqTh5AW+4wSzQVNvWSsZpUzw/WCM4F4l26dirPEFIp6Cb5bKhmjnmDdaepNYwS1lFbL1GabTr093g+9QaVS6cZCNHcQglxl4dtjDxTK7Q2iMT1v6lde6IIcDW/d8yDy5adt5xq/tEJbUbLShWG8FVE2lVc7mFoo6kaz6S/u7m6Bgm5UNVt83r+zHFQhlbldHybv70lpJ+/C7t5nJCkJoIDqVY6cGRa/C3yj5E6GApHSUvYW5G1T95d3Ow6XhV/KhSyF6shEpjNfDmtPu0Mhza16ntf+0+/Y6Gi+UqnVDmIh1emCV7mdbSk0huWyN1nrdcsUb3+tVwfF2qBzrSHUYU2gaVJvx4KsKNJFQ9cP5gRyCjPTS8u5NKuKjCByDJk7rdJoFguS9JahcG7o+yCo+99ZqpLOTA0NN5ixiMhJwqJeAsMsrf+qsjGghqscq7HJdq9z6oiRsDkoV2reTjZqiNtliJO3FqlgWK+E+9DRrtZBEN6p2VWvEg5uMmQ0NduuQAj3Wag6qn3JcNQXZ2djqBF6c1nkl0w+x8WFmaoL6wmWp3/BMJO2LhtCQpb0ZimY0zTmzNB1txxpFjY9ofraUBAOwXAw2IHd8DZstGpOUE1rDIaDcRKZDXHDg9T0NgS6DmFJbgim0PHAsN2DnfHEuymGKdhMNdah8Q8PVUl+w7CUPxI0JSm6YFglMpeLlyeBDyK+QQ0vxJBVMnEAa/SCoUENqfysembY1CcW7NCRBll6ZsiYqiBsQzUZeLGm8NAxmNBJ8Hw8rpUr5U3GbEnsEBZbLexxsA6hsnprRNGmhl4HVvfzn2NIFO7Eg8IVfi/ys4p6KUtLUC4VISkarp6vKnJ2d0S3inyx2v/Gv5ylrEriyzE0DJqnursVy9G5YbOYjQRoibKL7pmhaqoMQw3LXpxOyzwEkGUITxIaw/Lmkh1JzNCDJen12PS5YYp5Zago4s8xTCsat9GGxK60C4xtQtN0OUshqHFehwkvR3a81XQhN/uxYIFrUwfD9I8xTMcTo3Sh0vh+8Tsfdkj339lImRqW9IA1GzIvi2cxTMBQUcghNYQsTcm8OBwWHJFRuQNqeLAk5Qjp1anYUNXAMATDNKN2BlBy2iemJj5v37QO4YZU4yGGZZr331otmxd/NPRhFvqpQvipoX8n4nefuM3SaFe0Zy2IodEsFhTGKULFzftzOWuZbp3UkPxnHgrtKDi1Vpuw+PKFlmqLxRIoBqIE7VIkLsLOYQSJ1hBlvgGzrNW8fVE1ub0x9DXtLznusE03gw1OY61j+HSwEC+xK9QaYshnqGEF3inq83a5Xm9fa8ikNYmN54/Dp7VwoSear25KowhimNfzQ5Mx51x35LpVwu4+gQ1ydJrlsnvrRtPQg4JpJhOXbvSj5fUS3Q30qirl+i5QPBLVXbcJaf2Ybi+TEnyjKpotaBOFKjRFRjDXYtgGL8ddcAjvHzrJ8AAyEooJQ5zpse5GXDjsQjcZ7kOZX4EmBwwVWzuZZukJ1Kbh4GmtVl+5zpAl6Qwhm8e34AcK54e5j8+7WBMM6eyOMoyW5HW/qa8S9j9uQdL5j492oceE1gbCqZlcAK0MRGoUwMKDOIHhNIbN/GnUmlunS7G4S2yOdkE+NYRKwq3Stg+q7MO7dx/yTK8NG/543O22oRxUwoNEtrOHHu1f4BDtYSrdQkvOQAzpOlQ05QS6n9rgBNY+xDAMb4jh9KGFs/Bk634trM0XcvKZYoOHGLqufkRkOb5FDe9wjY+CZ1AenzR9w4eeBlbdqSBx/QDSFApnsbo8gjjBvibMUEP/FLrWnZFbggDPsVkwdEfVrCwTVrAW/WlH9F9fvHjx3w8lkerU6CYObVu96+RaNrE2QZfylLY7wyiSycqAlpw1QWA36Dr0TgRWOQHDite7zpAmZTqd+/NftmZCLzzuDpPU9BlGQy5M3PWRP0xB7Q5G66PRHaHRcv4YuLDHjUbVvWA0KU5mGElIVib5vKHnH+/1/SDIr7Ks1R9RCirPCAFsnH6xw0HnHYxGi5wsq9A3fD0aFeEO+JMXjx69+L2mihtdWgnK9dBb2HS4yOQlJundb3v1wWAQtg92hBY1XICQdtdYjum0KWsMo6wtDDxv/jpDWIX0hW+tLXx7uv/VuLwAZ9NDJLrX73f6M/dMtXX7qA+syUstOekswy1Sv7DUcPqdmf4wbWumRZP2zmnWfDnT788MTVsr0PM792yWNZ2No6NOv8OqHTrcc9geeFlSn9M/dnN/+PzRi0d3U7BFJieH347H483DghVF9MGYaTJJ5/BgPN7vFThNyUWy9nx1ZWWld88m5tphD3gJd0A7vem7m2IIhhLZOfjLytGBV/c2HcaWoPNOT+956SMLlVDOTmanXPh+Wp0eIdOHC/S0NNzDps7Oh49JavrdVyP8CNyGpuTfQpb+9eF5j8FMeT3u6yPpCzO9/I6G4jq712hmxMW9+fDx8fj72riTM2f5VCZz1Znkrbm+feSNT688DoI8Lz/85O5DSf7bRr7xEpdQoV2KBOfwT96g9v2thR/WLELeeDb0kzeZVz5CuHj+T31XkoiYllRBvnl+P801E7sEA5eTpJYar3rz3vff7D7Yfgk3C5lf+Hkmgf5MzUUf5KnpNGXkhky4+PmD7sK2k7v3UpKUX/x5psynJemDPDU1I6gmKSjTtpRSk40H/7MdC/yVhn/zanlznOm/AsfABvl+R76ai7msZVIM9/LTT+/BzeP7Gf3vDU0hDCtwAv8h/5PvQ5KG+xnYhTTt2kd0/69RMtCKE9PUrtwPfw1kMhlFUTQNqvi7/A/U3zHUUNNSDMv8Wg2n/GrXIIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgyP8p/wv1tBAp9RwclQAAAABJRU5ErkJggg==)\n\n\nIn this Notebook, we will see some basic univariate, bivariate and mainly on the various methods to handle categorical variables in a Machine learning Pipeline as the dataset seems to have only categorical data."},{"metadata":{},"cell_type":"markdown","source":"
\n \n\n 1. [Descriptive Statistics](#desc)\n \n 1.1 [Univariate Analysis](#uva)\n \n 1.2 [Bivariate Analysis](#biva)\n\n \n 2. [Handling Categorical data](#hcd)\n \n 2.1 [Replace values](#rv)\n \n 2.2 [Label-encoding](#len)\n \n 2.3 [One-hot encoding](#oh)\n \n 2.4 [Binary Encoding](#bec)\n \n 2.5 [Backward Differencing encoding](#bde)\n \n \n 3. [NLP](#nl)\n \n 3.1 [Basic Analysis+Ngram](#bg)\n \n 3.2 [NER](#ner)\n \n 3.3 [POS](#pos)\n \n 3.4 [Text Readability](#tex)\n \n \n 4. [Boosting Models](#bst)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom nltk.tokenize import RegexpTokenizer\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/data-science-bowl-2019/train.csv')\ntest_df = pd.read_csv('../input/data-science-bowl-2019/test.csv')\ntrain_labels_df = pd.read_csv('../input/data-science-bowl-2019/train_labels.csv')\nspecs_df = pd.read_csv('../input/data-science-bowl-2019/specs.csv')\nsample_submission_df = pd.read_csv('../input/data-science-bowl-2019/sample_submission.csv')\n\ndata = pd.read_csv('/kaggle/input/jobs-on-naukricom/home/sdf/marketing_sample_for_naukri_com-jobs__20190701_20190830__30k_data.csv')\ndata.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"
\n \nBasically, the following data types can be used in base python:\n\n - boolean\n - integer\n - float\n - string\n - list\n - None\n - long\n - complex\n - object\n\n\nNumerical or Quantitative\n \n - Discrete\n Integer (int)\n - Continuous\n Float (float)\n\n\nCategorical or Qualitative\n - Nominal\n Boolean (bool)\n String (str)\n None (NoneType)\n - Ordinal\n Only defined by how you use the data\n Often important when creating visuals"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" \n\n### Univariate\n\nuni- means one and bi- means two: think of a unicycle, which has one wheel, and a bicycle, which has two. Multi means many and in statistics it is often used to mean “more than two.”\n\nUnivariate statistics such as the mean therefore describe characteristics of one variable, and the bar chart and histogram are examples of univariate graphic displays."},{"metadata":{},"cell_type":"markdown","source":"Frequency Table - A type of Univariate analysis and a common way to summarize categorical data\n\n#### Here lets look at Role Category"},{"metadata":{"trusted":true},"cell_type":"code","source":"rc = data['Role Category'].value_counts().reset_index()\nrc.columns = ['Role Category', 'Count']\nrc['Percent'] = rc['Count']/rc['Count'].sum() * 100\nrc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### I see a lot of 'sentences' mentioned as part of Role Category, which isnt ideal. \n\n#### Just checking the top 50 to see if they are proper, and they are but plotting the top ones"},{"metadata":{},"cell_type":"markdown","source":"Bar Chart - A type of Univariate analysis and a common way to visualize categorical data\n\nIn the X-Axis is the name and Y-axis has the frequency AKA the count of the Role Category."},{"metadata":{"trusted":true},"cell_type":"code","source":"rc = rc[:10]\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import rcParams\n\nrcParams['figure.figsize'] = 13, 10\nax = sns.barplot(x=\"Role Category\", y=\"Count\", data=rc)\nax.set_xticklabels(ax.get_xticklabels(), rotation=45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Sometimes, a pie chart could also help"},{"metadata":{"trusted":true},"cell_type":"code","source":"rc = data['Role Category'].value_counts().nlargest(n=10)\n\nfig = px.pie(rc, \n values = rc.values, \n names = rc.index, \n title=\"Top 10 Role Categories\", \n color=rc.values)\n \nfig.update_traces(opacity=0.5,\n marker_line_color='rgb(8,48,107)',\n marker_line_width=1.5)\nfig.update_layout(title_x=0.5)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### What can be observed is, nearly 40% is programming & Design job roles in naukri."},{"metadata":{},"cell_type":"markdown","source":"#### Top Locations"},{"metadata":{"trusted":true},"cell_type":"code","source":"location = data['Location'].value_counts().nlargest(n=10)\n\nfig = px.bar(y=location.values,\n x=location.index,\n orientation='v',\n color=location.index,\n text=location.values,\n color_discrete_sequence= px.colors.qualitative.Bold)\n\nfig.update_traces(texttemplate='%{text:.2s}', \n textposition='outside', \n marker_line_color='rgb(8,48,107)', \n marker_line_width=1.5, \n opacity=0.7)\n\nfig.update_layout(width=800, \n showlegend=False, \n xaxis_title=\"City\",\n yaxis_title=\"Count\",\n title=\"Top 10 cities by job count\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So, for categorical data;\n\n- Frequency Tables - Great for Numerical Summaries\n- Bar Charts - Great for Visualization"},{"metadata":{},"cell_type":"markdown","source":"#### QUANTITATIVE DATA - UNIVARIATE ANALYSIS"},{"metadata":{},"cell_type":"markdown","source":"Variables that have a numerical value(Quantity) that we can perform mathematical operations on.\n\nDivided into 2\n\n- Discrete: Age, Number of Children in a room etc.. \n- Continuous: Height, Weight etc..\n\n\n##### We dont have numerical variables here to plot, but ideal plots would be Histogram, boxplots."},{"metadata":{},"cell_type":"markdown","source":" \n\n### Bivariate\n\n\nTwo-way contingency table between Job Title and Job Experience required"},{"metadata":{"trusted":true},"cell_type":"code","source":"data1 = data[:10 ] ## taking just 10 records for demo\n\nlis_sum_t = data1[['Job Title', 'Job Experience Required']]\ntwo_cls = pd.crosstab(lis_sum_t['Job Title'], lis_sum_t['Job Experience Required'])\n\ntwo_cls.plot.bar(stacked=True)\n#plt.legend(title='mark')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" \n\n### Handling Categorical data\n\n\n1. Drop them entirely(if they dont contain any useful info) - Cant be done here.\n\n\n \n\n#### 2. Replacing values\n\n\n#### For instance, Job Experience required contains a lot of values that can be cut short to just 3 or 4 for better feature engineering. Lets do that! (I am using only a subset of the data but the same can be done for the entire dataset)"},{"metadata":{"trusted":true},"cell_type":"code","source":"place_map = {'Location': {'Hyderabad': 1, 'Pune': 2, 'Bengaluru': 3, 'Mumbai': 4,\n 'Gurgaon': 5, 'Pune,Pune': 6}}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = data1['Location'].astype('category').cat.categories.tolist()\n\nreplace_map_comp = {'Location' : {k: v for k,v in zip(labels,list(range(1,len(labels)+1)))}}\n\nprint(replace_map_comp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data1.replace(replace_map_comp, inplace=True)\ndata1['Location']\ndata1['Location'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" \n\n#### 3. Label Encoding\n\n![](https://i.imgur.com/tEogUAr.png)\n\nAnother approach is to encode categorical values with a technique called \"label encoding\", which allows you to convert each value in a column to a number. Numerical labels are always between 0 and n_categories-1.\n\nYou can do label encoding via attributes .cat.codes on your DataFrame's column or through sklearns label encoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlb_make = LabelEncoder()\n\ndata2 = data[:10]\n\ndata2['l_code'] = lb_make.fit_transform(data1['Location'])\n\ndata2.head() #Results in appending a new column to df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Label encoding is pretty much intuitive and straight-forward and may give you a good performance from your learning algorithm, but it has as disadvantage that the numerical values can be misinterpreted by the algorithm. Should the location Hyderabad be given more weight than others?\n\nTo solve this issue there is another popular way to encode the categories via something called one-hot encoding.\n\n\n \n\n#### 4. One-Hot encoding\n\n\n![](https://i.imgur.com/TW5m0aJ.png)\n\nThe basic strategy is to convert each category value into a new column and assign a 1 or 0 (True/False) value to the column. This has the benefit of not weighting a value improperly.\n\nThere are many libraries out there that support one-hot encoding but the simplest one is using pandas' .get_dummies() method.\n\nThis function is named this way because it creates dummy/indicator variables (1 or 0). There are mainly three arguments important here, the first one is the DataFrame you want to encode on, second being the columns argument which lets you specify the columns you want to do encoding on, and third, the prefix argument which lets you specify the prefix for the new columns that will be created after encoding."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\n\nlb = LabelBinarizer()\n\nlb_results = lb.fit_transform(data2['Location'])\n\nlb_results_df = pd.DataFrame(lb_results, columns=lb.classes_)\n\nprint(lb_results_df.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that this lb_results_df resulted in a new DataFrame with only the one hot encodings for the feature carrier. This needs to be concatenated back with the original DataFrame, which can be done via pandas' .concat() method. The axis argument is set to 1 as you want to merge on columns."},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df = pd.concat([data2, lb_results_df], axis=1)\n\nresult_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" \n\n#### 5. Binary Encoding\n\nIn this technique, first the categories are encoded as ordinal, then those integers are converted into binary code, then the digits from that binary string are split into separate columns. \n\nThis encodes the data in fewer dimensions than one-hot.\n\nYou can do binary encoding via a number of ways but the simplest one is using the category_encoders library. You can install category_encoders via pip install category_encoders (shown below)\n\nYou have to first import the category_encoders library after installing it. Invoke the BinaryEncoder function by specifying the columns you want to encode and then call the .fit_transform() method on it with the DataFrame as the argument."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install category_encoders","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\n\nencoder = ce.BinaryEncoder(cols=['Location'])\ndf_binary = encoder.fit_transform(data2)\ndf_binary.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Notice that four new columns are created in place of the Location column with binary encoding for each location in the feature."},{"metadata":{},"cell_type":"markdown","source":" \n\n#### 6. Backward Difference Encoding\n\nThis technique falls under the contrast coding system for categorical features.\n\nA feature of K categories, or levels, usually enters a regression as a sequence of K-1 dummy variables. In backward difference coding, the mean of the dependent variable for a level is compared with the mean of the dependent variable for the prior level. This type of coding may be useful for a nominal or an ordinal variable. We will apply this to Job Title"},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = ce.BackwardDifferenceEncoder(cols=['Job Title'])\n\ndf_bd = encoder.fit_transform(data2)\n\ndf_bd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"The interesting thing here is that you can see that the results are not the standard 1’s and 0’s you saw in the dummy encoding examples but rather regressed continuous values."},{"metadata":{},"cell_type":"markdown","source":"#### 7. Other feature engineering approach for Features like Job Experience"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"##### Miscellaneous Features\n\nSometimes you may encounter categorical feature columns which specify the ranges of values for observation points,\nfor example, the age column might be described in the form of categories like 0-20, 20-40 and so on.\n\nWhile there can be a lot of ways to deal with such features,\nthe most common ones are either split these ranges into two separate columns or replace\nthem with some measure like the mean of that range."},{"metadata":{"trusted":true},"cell_type":"code","source":"data2['Job Experience Required'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data2['Job Experience Required'] = data2['Job Experience Required'].str.replace(\"yrs\", \"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_mean(x):\n split_list = x.split('-')\n mean = (float(split_list[0])+float(split_list[1]))/2\n return mean\n\ndata2['exp_mean'] = data2['Job Experience Required'].apply(lambda x: split_mean(x))\n\ndata2.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Columns like the above will help in building better models that take the Engineered features.\n\n \n\n\n### We Will move to NLP"},{"metadata":{},"cell_type":"markdown","source":" \n\n### Exploratory Data Analysis for Natural Language Processing"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"data[\"About_me\"] = \"I am a\" + data[\"Job Title\"]+\" from \"+ data[\"Location\"]+\". \"+\" I work in the \"+ data[\"Industry\"]+\" Industry as a \"+ data['Role']\ndata['len'] = data['About_me'].str.len()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Common packages for NLP\n\n - re\n - Spacy\n - NLTK\n - gensim"},{"metadata":{"trusted":true},"cell_type":"code","source":"import spacy\nnlp = spacy.load(\"en_core_web_sm\")\nimport re\nimport nltk\nimport gensim","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Analysing basic text stats\n\n#### Text statistics visualizations are simple but very insightful techniques. \n\n - word frequency analysis,\n - sentence length analysis,\n - average word length analysis"},{"metadata":{},"cell_type":"markdown","source":"##### First, I’ll take a look at the number of characters present in each sentence. This can give us a rough idea about each persons summary"},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.express as px\nfig = px.histogram(data, x=\"len\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Very skewed plot. But Majorly length is ~200 range.\n\n\n#### So there is a huge lengthy 'about me'. lets quickly see that"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_colwidth', -1)\nm = data['len'].max()\nab = data[data['len'] == m]\nab['About_me']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Average word length in each sentence."},{"metadata":{"trusted":true},"cell_type":"code","source":"import re \n\ndata['About_me'] = data['About_me'].fillna('').astype('str')\ndata['detail_abt'] = data['About_me'].apply(lambda x: nltk.sent_tokenize(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_len = data['About_me'].str.split().apply(lambda x : [len(i) for i in x]).map(lambda x: np.mean(x))\n\nimport plotly.express as px\n\nfig = px.histogram(avg_len)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The average word length ranges between 4 to 6 with 5 being the most common length.\n\nCould stopwords have played a role here? Lets see that."},{"metadata":{},"cell_type":"markdown","source":"#### To get the corpus containing stopwords you can use the nltk library.\n\n#### Nltk contains stopwords from many languages. Since we are only dealing with English news I will filter the English stopwords from the corpus."},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nstopwords = nltk.corpus.stopwords.words('english')\n#stop=set(stopwords.words('english'))\nprint(stopwords[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now, we’ll create the corpus.\n\ncorpus=[]\n\nnew = data['About_me'].str.split()\n\nnew = new.values.tolist()\n\ncorpus=[word for i in new for word in i]\n\nfrom collections import defaultdict, Counter\n\ndic=defaultdict(int)\n\nfor word in corpus:\n if word in stopwords:\n dic[word]+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plotly.offline.initnotebookmode(connected = True)\n\ntop = sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10]\n\nx,y = zip(*top)\n\nx = list(x)\ny = list(y)\n\nimport plotly.graph_objects as go\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n textposition='auto',\n marker=dict(color='rgba(58, 71, 80, 0.6)')))\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can evidently see that stopwords such as “a”,” in” and “the” dominate the about me column.\n\nApplying to the corpus now"},{"metadata":{"trusted":true},"cell_type":"code","source":"counter=Counter(corpus)\nmost=counter.most_common()\n\nx, y= [], []\n\nfor word,count in most[:40]:\n if (word not in stopwords):\n x.append(word)\n y.append(count)\n \n\nx = list(x)\ny = list(y)\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n marker=dict(color='rgba(246, 78, 139, 0.6)')))\n\nfig.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ngram exploration"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### Ngrams are simply contiguous sequences of n words. For example “riverbank”,” The three musketeers” etc.If the number of words is two, it is called bigram. For 3 words it is called a trigram and so on.\n\n#### Looking at most frequent n-grams can give you a better understanding of the context in which the word was used.\n\n#### To build a representation of our vocabulary we will use Countvectorizer. Countvectorizer is a simple method used to tokenize, vectorize and represent the corpus in an appropriate form. It is available in sklearn.feature_engineering.text\n\n#### So with all this, we will analyze the top bigrams in our dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.util import ngrams\nfrom sklearn.feature_extraction.text import CountVectorizer\n\ndef get_top_ngram(corpus, n=None):\n vec = CountVectorizer(ngram_range=(n, n)).fit(corpus)\n bag_of_words = vec.transform(corpus)\n sum_words = bag_of_words.sum(axis=0) \n words_freq = [(word, sum_words[0, idx]) \n for word, idx in vec.vocabulary_.items()]\n words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n return words_freq[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_n_bigrams=get_top_ngram(data['About_me'],2)[:10]\n\nx,y=map(list,zip(*top_n_bigrams))\n\n#sns.barplot(x=y,y=x)\n\nx = list(x)\ny = list(y)\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n marker_color='rgb(26, 118, 255)'))\n\nfig.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### How about trigrams?"},{"metadata":{"trusted":true},"cell_type":"code","source":"top_tri_grams=get_top_ngram(data['About_me'], n=3)\nx,y=map(list,zip(*top_tri_grams))\n\n#sns.barplot(x=y,y=x)\n\nx = list(x)\ny = list(y)\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h'))\nfig.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We can see the top tri-grams like 'work-in-the', 'software-services-industry' etc.. There are few stopwords which can be further removed. "},{"metadata":{},"cell_type":"markdown","source":"### Wordcloud\n\n\nWordcloud is a great way to represent text data. The size and color of each word that appears in the wordcloud indicate it’s frequency or importance."},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data):\n wordcloud = WordCloud(\n background_color='white',\n stopwords=stopwords,\n max_words=100,\n max_font_size=30,\n scale=3,\n random_state=1)\n \n wordcloud=wordcloud.generate(str(data))\n\n fig = plt.figure(1, figsize=(16, 12))\n plt.axis('off')\n\n plt.imshow(wordcloud)\n plt.show()\n\nshow_wordcloud(corpus)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### More Frequency for words like Services, Industry, Software etc."},{"metadata":{},"cell_type":"markdown","source":" \n\n### Named Entity Recognition(NER)\n\n\nWhat is NER? \n\n\nNamed entity recognition is an information extraction method in which entities that are present in the text are classified into predefined entity types like “Person”,” Place”,” Organization”, etc. By using NER we can get great insights about the types of entities present in the given text dataset.\n\nCommon Libraries for NER:\n\n - Standford NER\n - spaCy\n - NLTK\n \nWe will use Spacy which is a fantastic library. You can read their documentation [here](https://spacy.io/)\n\nBesides NER, spaCy provides many other functionalities like pos tagging, word to vector transformation, etc.\nSpaCy’s named entity recognition has been trained on the OntoNotes 5 corpus and it supports the following entity types:\n\n\n![](https://miro.medium.com/max/1824/1*qQggIPMugLcy-ndJ8X_aAA.png)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# There are three pre-trained models for English in spaCy. I will use en_core_web_sm for our task but you can try other models.\n\nimport spacy\nnlp = spacy.load(\"en_core_web_sm\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Example Spacy Output for a sample document"},{"metadata":{"trusted":true},"cell_type":"code","source":"doc = nlp('India and Iran have agreed to boost the economic viability \\\nof the strategic Chabahar port through various measures, \\\nincluding larger subsidies to merchant shipping firms using the facility, \\\npeople familiar with the development said on Thursday.')\n\n[(x.text,x.label_) for x in doc.ents]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### So Spacy is able to identify the entity of the words present in the sentence. India and Iran(GPE) are countries(refer the table above) and Chabahar is identified as a person although technically its a place/location and Thursday as date"},{"metadata":{"trusted":true},"cell_type":"code","source":"from spacy import displacy\n\ndisplacy.render(doc, style='ent')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we will run the named entity recognition on our data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ner(text):\n doc=nlp(text)\n return [X.label_ for X in doc.ents]\n\ndata1 = data[:1000]\n\nent = data1['About_me'].apply(lambda x : ner(x))\n\nent=[x for sub in ent for x in sub]\n\ncounter=Counter(ent)\n\ncount=counter.most_common()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualizing the entity frequency from our corpus"},{"metadata":{"trusted":true},"cell_type":"code","source":"x,y=map(list,zip(*count))\n\n#sns.barplot(x=y,y=x)\n\nx = list(x)\ny = list(y)\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n marker_color='rgb(55, 83, 109)'))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SO we have more GPE Entities than any other. We can also visualize the most common tokens per entity."},{"metadata":{"trusted":true},"cell_type":"code","source":"def ner(text,ent=\"GPE\"):\n doc=nlp(text)\n return [X.text for X in doc.ents if X.label_ == ent]\n\ngpe = data1['About_me'].apply(lambda x: ner(x))\n\ngpe=[i for x in gpe for i in x]\n\ncounter=Counter(gpe)\n\nx,y=map(list,zip(*counter.most_common(10)))\n\n# sns.barplot(y,x)\n\nx = list(x)\ny = list(y)\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n ))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### So the most common GPE is Mumbai and Spacy has been spot on for now! "},{"metadata":{},"cell_type":"markdown","source":" \n\n### Exploration through Parts of Speech Tagging in python\n\nParts of speech (POS) tagging is a method that assigns part of speech labels to words in a sentence. There are eight main parts of speech:\n\n - Noun (NN)- Joseph, London, table, cat, teacher, pen, city\n - Verb (VB)- read, speak, run, eat, play, live, walk, have, like, are, is\n - Adjective(JJ)- beautiful, happy, sad, young, fun, three\n - Adverb(RB)- slowly, quietly, very, always, never, too, well, tomorrow\n - Preposition (IN)- at, on, in, from, with, near, between, about, under\n - Conjunction (CC)- and, or, but, because, so, yet, unless, since, if\n - Pronoun(PRP)- I, you, we, they, he, she, it, me, us, them, him, her, this\n - Interjection (INT)- Ouch! Wow! Great! Help! Oh! Hey! Hi!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# I will use the nltk to do the parts of speech tagging but there are other libraries that do a good job (spacy, textblob).\n\n# Let’s look at an example.\n\nimport nltk\n\nsentence=\"The greatest comeback stories in 2019\"\n\ntokens = nltk.word_tokenize(sentence)\n\nnltk.pos_tag(tokens)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Spacy helps in visualising POS as well"},{"metadata":{"trusted":true},"cell_type":"code","source":"doc = nlp('The greatest comeback stories in 2020')\n\ndisplacy.render(doc, style='dep', jupyter=True, options={'distance': 90})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can observe various dependency tags here. For example, DET tag denotes the relationship between the determiner “the” and the noun “stories”. You can read more [here](https://universaldependencies.org/u/dep/index.html)\n\n\nSo how does it show for our data? Lets see"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pos(text):\n pos=nltk.pos_tag(nltk.word_tokenize(text))\n pos = list(map(list, zip(*pos)))[1] \n return pos\n\ndata1 = data[:1000]\n\ndata1['About_me'] = data1['About_me'].str.replace('', 'dummy')\n\ntags = data1['About_me'].apply(lambda x : pos(x))\n\ntags=[x for l in tags for x in l]\ncounter=Counter(tags)\nx,y=list(map(list,zip(*counter.most_common(7))))\n\n# sns.barplot(x=y,y=x)\n\nx = list(x)\ny = list(y)\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n marker_color='rgb(55, 83, 109)'))\nfig.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can clearly see that the noun (NN) dominates in our about me followed by the adjective (JJ).\n\nWe can also see which singular noun occurs the most"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_adjs(text):\n adj=[]\n pos=nltk.pos_tag(nltk.word_tokenize(text))\n for word,tag in pos:\n if tag=='NN':\n adj.append(word)\n return adj\n\n\ndata2 = data[:1000]\n\nwords = data2['About_me'].apply(lambda x : get_adjs(x))\n\nwords=[x for l in words for x in l]\ncounter=Counter(words)\n\nx,y=list(map(list,zip(*counter.most_common(7))))\n\n# sns.barplot(x=y,y=x)\n\nx = list(x)\ny = list(y)\n\nfig = go.Figure(go.Bar(\n x=y,\n y=x,\n orientation='h',\n marker_color='rgb(227, 119, 194)'))\nfig.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### So Nouns such as Executive, Project, officer dominate our data"},{"metadata":{},"cell_type":"markdown","source":" \n\n### Text Complexity/ Readability\n\n\nIt can be very informative to know how readable the text is and what type of reader can fully understand it.\n\nYou can actually put a number called readability index on a document or text. Readability index is a numeric value that indicates how difficult (or easy) it is to read and understand a text.\n\nThere are many readability score formulas available for the English language. Some of the most prominent ones are:\n\n\n![](https://github.com/rakash/images/blob/master/text_read.png?raw=true)\n\n\n\nTextstat is a cool Python library that provides an implementation of all these text statistics calculation methods.\n\nLet’s use Textstat to implement Flesch Reading Ease index. More on the package [here](https://pypi.org/project/textstat/)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install textstat\n\nfrom textstat import flesch_reading_ease\n\nscore = data2['About_me'].apply(lambda x : flesch_reading_ease(x))\n\nscore.hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Our Scores are majorly in the range of 20-40, meaning the readability of the descriptions of each person is not good. So do you need a PhD or something to read those? No, Better cleaning up / preprocessing would do I guess. But thats for later. Perhaps you can try this approach in your future projects. \n\n### Lets see what a normalized histogram gives us"},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.graph_objects as go\nimport numpy as np\n\n## Normalized Histogram\n\nfig = go.Figure(data=[go.Histogram(x=score, histnorm='probability')])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let’s check all about me descriptions that have a readability score below 5."},{"metadata":{"trusted":true},"cell_type":"code","source":"data2['read_score'] = data2['About_me'].apply(lambda x : flesch_reading_ease(x))\n\nalls = data2[data2['read_score'] < 5].head(5)\n\nalls['About_me']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### So, hopefully, you will find some of the above useful in your current and future projects.\n\n\n### Do upvote and check out the reference kernels that helped me \n \n https://www.kaggle.com/datafan07/disaster-tweets-nlp-eda-bert-with-transformers/\n \n https://www.kaggle.com/gunesevitan/nlp-with-disaster-tweets-eda-cleaning-and-bert/"},{"metadata":{},"cell_type":"markdown","source":" \n\n### What is Boosting?\n\n\nBoosting (originally called hypothesis boosting) refers to any Ensemble method that can combine several weak learners into a strong learner. The general idea of most boosting methods is to train predictors sequentially, each trying to correct its predecessor. There are many boosting methods available, but by far the most popular are AdaBoost(short for Adaptive Boosting) and Gradient Boosting. We will talk about both here, but after reading in the data and pre-processing them.\n\n![](https://miro.medium.com/max/694/1*QJZ6W-Pck_W7RlIDwUIN9Q.jpeg)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost\nimport gc\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier \nimport lightgbm as lgb\nfrom numba import jit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_time_features(df):\n df['timestamp'] = pd.to_datetime(df['timestamp'])\n df['date'] = df['timestamp'].dt.date\n df['month'] = df['timestamp'].dt.month\n df['hour'] = df['timestamp'].dt.hour\n df['year'] = df['timestamp'].dt.year\n df['dayofweek'] = df['timestamp'].dt.dayofweek\n df['weekofyear'] = df['timestamp'].dt.weekofyear\n df['dayofyear'] = df['timestamp'].dt.dayofyear\n df['quarter'] = df['timestamp'].dt.quarter\n df['is_month_start'] = df['timestamp'].dt.is_month_start \n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_object_columns(df, columns):\n df = df.groupby(['installation_id', columns])['event_id'].count().reset_index()\n df = df.pivot_table(index = 'installation_id', columns = [columns], values = 'event_id')\n df.columns = list(df.columns)\n df.fillna(0, inplace = True)\n return df\n\ndef get_numeric_columns(df, column):\n df = df.groupby('installation_id').agg({f'{column}': ['mean', 'sum', 'min', 'max', 'std', 'skew']})\n df[column].fillna(df[column].mean(), inplace = True)\n df.columns = [f'{column}_mean', f'{column}_sum', f'{column}_min', f'{column}_max', f'{column}_std', f'{column}_skew']\n return df\n\ndef get_numeric_columns_add(df, agg_column, column):\n df = df.groupby(['installation_id', agg_column]).agg({f'{column}': ['mean', 'sum', 'min', 'max', 'std', 'skew']}).reset_index()\n df = df.pivot_table(index = 'installation_id', columns = [agg_column], values = [col for col in df.columns if col not in ['installation_id', 'type']])\n df[column].fillna(df[column].mean(), inplace = True)\n df.columns = list(df.columns)\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def perform_features_engineering(train_df, test_df, train_labels_df):\n print(f'Perform features engineering')\n numerical_columns = ['game_time']\n categorical_columns = ['type', 'world']\n\n comp_train_df = pd.DataFrame({'installation_id': train_df['installation_id'].unique()})\n comp_train_df.set_index('installation_id', inplace = True)\n comp_test_df = pd.DataFrame({'installation_id': test_df['installation_id'].unique()})\n comp_test_df.set_index('installation_id', inplace = True)\n\n test_df = extract_time_features(test_df)\n train_df = extract_time_features(train_df)\n\n for i in numerical_columns:\n comp_train_df = comp_train_df.merge(get_numeric_columns(train_df, i), left_index = True, right_index = True)\n comp_test_df = comp_test_df.merge(get_numeric_columns(test_df, i), left_index = True, right_index = True)\n \n for i in categorical_columns:\n comp_train_df = comp_train_df.merge(get_object_columns(train_df, i), left_index = True, right_index = True)\n comp_test_df = comp_test_df.merge(get_object_columns(test_df, i), left_index = True, right_index = True)\n \n for i in categorical_columns:\n for j in numerical_columns:\n comp_train_df = comp_train_df.merge(get_numeric_columns_add(train_df, i, j), left_index = True, right_index = True)\n comp_test_df = comp_test_df.merge(get_numeric_columns_add(test_df, i, j), left_index = True, right_index = True)\n \n \n comp_train_df.reset_index(inplace = True)\n comp_test_df.reset_index(inplace = True)\n \n labels_map = dict(train_labels_df.groupby('title')['accuracy_group'].agg(lambda x:x.value_counts().index[0]))\n \n labels = train_labels_df[['installation_id', 'title', 'accuracy_group']]\n \n labels['title'] = labels['title'].map(labels_map)\n \n comp_test_df['title'] = test_df.groupby('installation_id').last()['title'].map(labels_map).reset_index(drop = True)\n \n comp_train_df = labels.merge(comp_train_df, on = 'installation_id', how = 'left')\n print('We have {} training rows'.format(comp_train_df.shape[0]))\n \n return comp_train_df, comp_test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### The metric used to evaluate the below models are quadratic kappa.\n\n#### [This kernel](https://www.kaggle.com/aroraaman/quadratic-kappa-metric-explained-in-5-simple-steps) explains more on the theory part as well."},{"metadata":{"trusted":true},"cell_type":"code","source":"def qwk3(a1, a2, max_rat=3):\n assert(len(a1) == len(a2))\n a1 = np.asarray(a1, dtype=int)\n a2 = np.asarray(a2, dtype=int)\n hist1 = np.zeros((max_rat + 1, ))\n hist2 = np.zeros((max_rat + 1, ))\n o = 0\n for k in range(a1.shape[0]):\n i, j = a1[k], a2[k]\n hist1[i] += 1\n hist2[j] += 1\n o += (i - j) * (i - j)\n e = 0\n for i in range(max_rat + 1):\n for j in range(max_rat + 1):\n e += hist1[i] * hist2[j] * (i - j) * (i - j)\n e = e / a1.shape[0]\n return 1 - o / e","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Adaboost\n\n\nOne way for a new predictor to correct its predecessor is to pay a bit more attention to the training instances that the predecessor underfitted. This results in new predictors focusing more and more on the hard cases. This is the technique used by Ada‐Boost. For example, to build an AdaBoost classifier, a first base classifier (such as a Decision Tree) is trained and used to make predictions on the training set. The relative weight of misclassified training instances is then increased. A second classifier is trained using the updated weights and again it makes predictions on the training set, weights are updated, and so on ...\n\n![](http://www.github.com/rakash/images1/blob/master/adaboost.jpg?raw=true)\n\n\nLet us see how decision boundaries are drawn for all the models for adaboost\n\n\n![](http://www.github.com/rakash/images1/blob/master/adaboost_db.jpg?raw=true)\n\n\nThe first classifier(notified by the line) gets many instances wrong, so their weights get boosted. The second classifier therefore does a better job on these instances, and so on. The plot on the right represents the same sequence of predictors except that the learning rate is halved (i.e., the misclassified instance weights are boosted half as much at every iteration). As you can see, this sequential learning technique has some similarities with Gradient Descent, except that instead of tweaking a single predictor’s parameters to minimize a cost function, AdaBoost adds predictors to the ensemble,gradually making it better.\n\nOnce all predictors are trained, the ensemble makes predictions very much like bagging or pasting, except that predictors have different weights depending on their overall accuracy on the weighted training set."},{"metadata":{"trusted":true},"cell_type":"code","source":"ada_train_df, ada_test_df = perform_features_engineering(train_df, test_df, train_labels_df)\n\nnull_columns = ada_test_df.columns[ada_test_df.isnull().any()]\nada_test_df[null_columns].isnull().sum()\n\nada_test_df['game_time_std'] = ada_test_df['game_time_std'].fillna(0)\nada_test_df['game_time_skew'] = ada_test_df['game_time_skew'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Below is the model function. As you can see, for ada boost we will be using a simple decision tree as the base estimator.\n\nLike Random Forest, AdaBoost makes predictions by applying multiple decision trees to every sample and combining the predictions made by individual trees. However, rather than taking the average of the predictions made by each decision tree in the forest (or majority in the case of classification), in the AdaBoost algorithm, every decision tree contributes a varying amount to the final prediction."},{"metadata":{"trusted":true},"cell_type":"code","source":"def adaboost_it(ada_train_df, ada_test_df):\n print(\"Ada-Boosting...\")\n t_splits = 5\n k_scores = []\n kf = KFold(n_splits = t_splits)\n features = [i for i in ada_train_df.columns if i not in ['accuracy_group', 'installation_id']]\n target = 'accuracy_group'\n oof_pred = np.zeros((len(ada_train_df), 4))\n y_pred = np.zeros((len(ada_test_df), 4))\n for fold, (tr_ind, val_ind) in enumerate(kf.split(ada_train_df)):\n print(f'Fold: {fold+1}')\n x_train, x_val = ada_train_df[features].iloc[tr_ind], ada_train_df[features].iloc[val_ind]\n y_train, y_val = ada_train_df[target][tr_ind], ada_train_df[target][val_ind]\n \n ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), n_estimators=200,algorithm=\"SAMME.R\", learning_rate=0.5)\n ada_clf.fit(x_train, y_train)\n oof_pred[val_ind] = ada_clf.predict_proba(x_val)\n \n y_pred += ada_clf.predict_proba(ada_test_df[features]) / t_splits\n \n val_crt_fold = qwk3(y_val, oof_pred[val_ind].argmax(axis = 1))\n print(f'Fold: {fold+1} quadratic weighted kappa score: {np.round(val_crt_fold,4)}')\n \n res = qwk3(ada_train_df['accuracy_group'], oof_pred.argmax(axis = 1))\n print(f'Quadratic weighted score: {np.round(res,4)}')\n \n return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = adaboost_it(ada_train_df, ada_test_df)\n\nada_test_df = ada_test_df.reset_index()\nada_test_df = ada_test_df[['installation_id']]\nada_test_df['accuracy_group'] = y_pred.argmax(axis = 1)\nada_sample_submission_df = sample_submission_df.merge(ada_test_df, on = 'installation_id')\nada_sample_submission_df.to_csv('ada_boost_submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### XGBoost\n\n\n![](https://miro.medium.com/max/583/1*FLshv-wVDfu-i54OqvZdHg.png)\n\n\nAnother very popular Boosting algorithm is Gradient Boosting. Just like AdaBoost, Gradient Boosting works by sequentially adding predictors to an ensemble, each one correcting its predecessor. However, instead of tweaking the instance weights at every iteration like AdaBoost does, this method tries to fit the new predictor to the residual errors made by the previous predictor.\n\nThink of XGBoost as gradient boosting on ‘steroids’ (well it is called ‘Extreme Gradient Boosting’ for a reason!). It is a perfect combination of software and hardware optimization techniques to yield superior results using less computing resources in the shortest amount of time."},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_train_df, xgb_test_df = perform_features_engineering(train_df, test_df, train_labels_df)\n\nfeatures = [i for i in xgb_train_df.columns if i not in ['accuracy_group', 'installation_id']]\ntarget = 'accuracy_group'\n\n\nx_train = xgb_train_df[features]\ny_train = xgb_train_df[target]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Grid search is very time consuming and therefore i have commented it for now.\n\n#from sklearn.model_selection import GridSearchCV\n#model = xgboost.XGBClassifier()\n\n#param_dist = {\"max_depth\": [10,30,50],\"min_child_weight\" : [1,3,6],\n # \"n_estimators\": [200],\n # \"learning_rate\": [0.05, 0.1,0.16],}\n\n#grid_search = GridSearchCV(model, param_grid=param_dist, cv = 3, verbose=10, n_jobs=-1)\n#grid_search.fit(x_train, y_train)\n#grid_search.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def xgb(xgb_train_df, xgb_test_df):\n print(\"XG-Boosting...\")\n t_splits = 5\n k_scores = []\n kf = KFold(n_splits = t_splits)\n features = [i for i in xgb_train_df.columns if i not in ['accuracy_group', 'installation_id']]\n target = 'accuracy_group'\n oof_pred = np.zeros((len(xgb_train_df), 4))\n y_pred = np.zeros((len(xgb_test_df), 4))\n for fold, (tr_ind, val_ind) in enumerate(kf.split(xgb_train_df)):\n print(f'Fold: {fold+1}')\n x_train, x_val = xgb_train_df[features].iloc[tr_ind], xgb_train_df[features].iloc[val_ind]\n y_train, y_val = xgb_train_df[target][tr_ind], xgb_train_df[target][val_ind]\n \n xgb_clf = xgboost.XGBClassifier()\n xgb_clf.fit(x_train, y_train)\n oof_pred[val_ind] = xgb_clf.predict_proba(x_val)\n \n y_pred += xgb_clf.predict_proba(xgb_test_df[features]) / t_splits\n \n val_crt_fold = qwk3(y_val, oof_pred[val_ind].argmax(axis = 1))\n print(f'Fold: {fold+1} quadratic weighted kappa score: {np.round(val_crt_fold,4)}')\n \n res = qwk3(xgb_train_df['accuracy_group'], oof_pred.argmax(axis = 1))\n print(f'Quadratic weighted score: {np.round(res,4)}')\n \n return y_pred\n\n\ny_pred = xgb(xgb_train_df, xgb_test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_test_df = xgb_test_df.reset_index()\nxgb_test_df = xgb_test_df[['installation_id']]\nxgb_test_df['accuracy_group'] = y_pred.argmax(axis = 1)\nxgb_sample_submission_df = sample_submission_df.merge(xgb_test_df, on = 'installation_id')\nxgb_sample_submission_df.to_csv('xgb_submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_sample_submission_df = xgb_sample_submission_df.drop('accuracy_group_x', axis=1)\nxgb_sample_submission_df.columns = ['installation_id', 'accuracy_group']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_sample_submission_df.to_csv('xgb_submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Catboost\n\n\nCatboost yields state-of-the-art results without extensive data training typically required by other machine learning methods, and it Provides powerful out-of-the-box support for the more descriptive data formats that accompany many business problems.\n\nMajor advantage is it handles categorical variables automatically, that is why the name 'CAT-boost'\n\nYou can know more about it [here](https://www.youtube.com/watch?time_continue=2&v=s8Q_orF4tcI)"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_train_df, cat_test_df = perform_features_engineering(train_df, test_df, train_labels_df)\n\nxc_train = cat_train_df[features]\nyc_train = cat_train_df[target]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cat_test_df.columns\n#import re\n\n# cat_test_df = cat_test_df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))\n#cat_train_df.columns = [\"\".join (c if c.isalnum() else \"_\" for c in str(x)) for x in cat_train_df.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import catboost as cb\n\ndef cat(cat_train_df, cat_test_df):\n print(\"Meeowwww...\")\n t_splits = 3\n k_scores = []\n kf = KFold(n_splits = t_splits)\n features = [i for i in cat_train_df.columns if i not in ['accuracy_group', 'installation_id']]\n target = 'accuracy_group'\n oof_pred = np.zeros((len(cat_train_df), 4))\n y_pred = np.zeros((len(cat_test_df), 4))\n for fold, (tr_ind, val_ind) in enumerate(kf.split(cat_train_df)):\n print(f'Fold: {fold+1}')\n x_train, x_val = cat_train_df[features].iloc[tr_ind], cat_train_df[features].iloc[val_ind]\n y_train, y_val = cat_train_df[target][tr_ind], cat_train_df[target][val_ind]\n \n cat_clf = cb.CatBoostClassifier(depth=10, iterations= 200, l2_leaf_reg= 9, learning_rate= 0.15, silent=True)\n cat_clf.fit(xc_train, yc_train)\n oof_pred[val_ind] = cat_clf.predict_proba(x_val)\n \n y_pred += cat_clf.predict_proba(cat_test_df[features]) / t_splits\n \n val_crt_fold = qwk3(y_val, oof_pred[val_ind].argmax(axis = 1))\n print(f'Fold: {fold+1} quadratic weighted kappa score: {np.round(val_crt_fold,4)}')\n \n res = qwk3(cat_train_df['accuracy_group'], oof_pred.argmax(axis = 1))\n print(f'Quadratic weighted score: {np.round(res,4)}')\n \n return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_cat = cat(cat_train_df, cat_test_df)\n\n\ncat_test_df = cat_test_df.reset_index()\n\ncat_test_df = cat_test_df[['installation_id']]\ncat_test_df['accuracy_group'] = y_pred_cat.argmax(axis = 1)\n\ncat_sample_submission_df = sample_submission_df.merge(cat_test_df, on = 'installation_id')\ncat_sample_submission_df.to_csv('submission.csv', index = False)\n\ncat_sample_submission_df = cat_sample_submission_df.drop('accuracy_group_x', axis=1)\n\ncat_sample_submission_df.columns = ['installation_id', 'accuracy_group']\n\ncat_sample_submission_df.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LightGBM\n\n\nIt is based on decision tree algorithms, it splits the tree leaf wise with the best fit whereas other boosting algorithms split the tree depth wise or level wise rather than leaf-wise. So when growing on the same leaf in Light GBM, the leaf-wise algorithm can reduce more loss than the level-wise algorithm and hence results in much better accuracy which can rarely be achieved by any of the existing boosting algorithms. Also, it is surprisingly very fast, hence the word ‘Light’.\n\n\nTHIS IS HOW IT WORKS IN XGBOOST\n\n\n![](https://s3-ap-south-1.amazonaws.com/av-blog-media/wp-content/uploads/2017/06/11194110/leaf.png)\n\n\nHOW IT WORKS IN LIGHTGBM¶\n\n\n![](https://s3-ap-south-1.amazonaws.com/av-blog-media/wp-content/uploads/2017/06/11194227/depth.png)"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train_df, lgb_test_df = perform_features_engineering(train_df, test_df, train_labels_df)\n\nxl_train = lgb_train_df[features]\nyl_train = lgb_train_df[target]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\ndef lgbc(lgb_train_df, lgb_test_df):\n print(\"I'm so light you know...\")\n t_splits = 3\n k_scores = []\n kf = KFold(n_splits = t_splits)\n features = [i for i in lgb_train_df.columns if i not in ['accuracy_group', 'installation_id']]\n target = 'accuracy_group'\n oof_pred = np.zeros((len(lgb_train_df), 4))\n y_pred = np.zeros((len(lgb_test_df), 4))\n for fold, (tr_ind, val_ind) in enumerate(kf.split(lgb_train_df)):\n print(f'Fold: {fold+1}')\n x_train, x_val = lgb_train_df[features].iloc[tr_ind], lgb_train_df[features].iloc[val_ind]\n y_train, y_val = lgb_train_df[target][tr_ind], lgb_train_df[target][val_ind]\n \n lg = lgb.LGBMClassifier(silent=False)\n lg.fit(xl_train, yl_train)\n oof_pred[val_ind] = lg.predict_proba(x_val)\n \n y_pred += lg.predict_proba(lgb_test_df[features]) / t_splits\n \n val_crt_fold = qwk3(y_val, oof_pred[val_ind].argmax(axis = 1))\n print(f'Fold: {fold+1} quadratic weighted kappa score: {np.round(val_crt_fold,4)}')\n \n res = qwk3(lgb_train_df['accuracy_group'], oof_pred.argmax(axis = 1))\n print(f'Quadratic weighted score: {np.round(res,4)}')\n \n return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lgb_train_df.columns = [\"\".join (c if c.isalnum() else \"_\" for c in str(x)) for x in lgb_train_df.columns]\n# lgb_test_df.columns = [\"\".join (c if c.isalnum() else \"_\" for c in str(x)) for x in lgb_test_df.columns]\n\n#y_pred_lgb = lgbc(lgb_train_df, lgb_test_df)\n\n#lgb_test_df = lgb_test_df.reset_index()\n#lgb_test_df = lgb_test_df[['installation_id']]\n\n#lgb_test_df['accuracy_group'] = y_pred_lgb.argmax(axis = 1)\n\n#lgb_sample_submission_df = sample_submission_df.merge(lgb_test_df, on = 'installation_id')\n\n#lgb_sample_submission_df.to_csv('lgb_submission.csv', index = False)\n\n#lgb_sample_submission_df = lgb_sample_submission_df.drop('accuracy_group_x', axis=1)\n\n#lgb_sample_submission_df.columns = ['installation_id', 'accuracy_group']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = [['ada', 0.42], ['xgb', 0.44], ['cat', 0.65], ['lgb', 0.62]]\n\ndf = pd.DataFrame(data, columns = ['Model', 'Validation Kappa Score']) \n\nimport plotly.graph_objects as go\n\nfig = go.Figure()\nfig.add_trace(go.Bar(x=df['Model'], y=df['Validation Kappa Score'], marker_color='#FFD700'))\nfig.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}