{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

CommonLit Readability Prize EDA

\n
\n\nIn this notebook I'll perform some exploratory data analysis, trying to update it often. \n\n

Table of Contents

\n\n- [files available and some info](#files)\n- [train.csv](#train)\n- [test.csv](#test)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:08:53.598138Z","iopub.execute_input":"2021-05-20T10:08:53.598536Z","iopub.status.idle":"2021-05-20T10:08:53.607079Z","shell.execute_reply.started":"2021-05-20T10:08:53.598506Z","shell.execute_reply":"2021-05-20T10:08:53.605459Z"}}},{"cell_type":"markdown","source":"\n\n
Files available and some info
","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nfrom sklearn.linear_model import LinearRegression\nimport spacy\nnlp = spacy.load(\"en_core_web_sm\")\nimport time\nimport matplotlib\nimport matplotlib.pyplot as plt\nplt.style.use('fivethirtyeight') #Not a big fan of that website, but love their plots\n\nimport os\nroot_path = '/kaggle/input/commonlitreadabilityprize/'\nos.listdir(root_path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T14:10:52.561789Z","iopub.execute_input":"2022-07-15T14:10:52.562169Z","iopub.status.idle":"2022-07-15T14:10:53.745041Z","shell.execute_reply.started":"2022-07-15T14:10:52.562138Z","shell.execute_reply":"2022-07-15T14:10:53.743975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(root_path + \"/train.csv\")\ntest = pd.read_csv(root_path + \"/test.csv\")\nsample_submission = pd.read_csv(root_path + \"/sample_submission.csv\")\n\ndisplay(train.info())\nprint(\"\\n\\n\")\ndisplay(test.info())\nprint(\"\\n\\n\")\ndisplay(sample_submission.info())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:10:53.746936Z","iopub.execute_input":"2022-07-15T14:10:53.747367Z","iopub.status.idle":"2022-07-15T14:10:53.836143Z","shell.execute_reply.started":"2022-07-15T14:10:53.747324Z","shell.execute_reply":"2022-07-15T14:10:53.835174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.sample(), test.sample(), sample_submission.sample())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:10:53.838332Z","iopub.execute_input":"2022-07-15T14:10:53.838641Z","iopub.status.idle":"2022-07-15T14:10:53.866679Z","shell.execute_reply.started":"2022-07-15T14:10:53.838610Z","shell.execute_reply":"2022-07-15T14:10:53.865963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n
train.csv
","metadata":{}},{"cell_type":"code","source":"original_cols = ['id', 'url_legal', 'license', 'excerpt', 'target', 'standard_error']\n\ntrain_stats = (pd.concat([train[original_cols].apply(lambda x: x.nunique(), axis = 0)\n .rename(\"distinct_values\").to_frame(),\n train[original_cols].apply(lambda x: x.notna().sum(), axis = 0)\n .rename(\"not_nan_values\").to_frame()], 1)\n .reset_index().rename({'index': 'variable'}, axis = 1))\n\ntrain_stats['distinct_over_notnan_percentage'] = (train_stats['distinct_values']/train_stats['not_nan_values']).round(2)\n\nfig, ax = plt.subplots(1, 2, figsize = (14, 6), gridspec_kw={'width_ratios': [1.2, 1]})\n\nplt.style.use('fivethirtyeight')\n\nsns.set_context(rc = {'patch.linewidth': 2.0})\nsns.barplot(x = 'variable', \n y = 'distinct_values',\n data = train_stats,\n edgecolor = 'black',\n linewidth = 2,\n palette=\"pastel\",\n ax = ax[0])\n\nfor index, row in train_stats.iterrows():\n value = row.distinct_values\n ax[0].text(index, value+20, value, color='black', ha=\"center\", \n fontsize = 15, fontweight = 'bold')\n\nax[0].grid(True)\nax[0].legend(fontsize=18)\nax[0].set_title('Distinct values for each variable', fontsize = 18, fontweight = 'bold')\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\nax[0].set_xlabel('')\nax[0].set_xticklabels(ax[0].get_xticklabels(), rotation = 35, fontsize = 13, color = 'black')\nax[0].set_ylabel('distinct_values', fontsize = 18, color ='black')\nplt.subplots_adjust(hspace = 0.3)\n\nsns.barplot(x = 'variable', \n y = 'not_nan_values',\n data = train_stats,\n palette=\"pastel\",\n linewidth = 2,\n edgecolor = 'black',\n ax = ax[1])\n\nfor index, row in train_stats.iterrows():\n value = row.not_nan_values\n ax[1].text(index, value+20, value, color='black', ha=\"center\", fontsize = 15, fontweight = 'bold')\n \nax[1].grid(True)\nax[1].set_title('Not NaN values for each variable', fontsize = 18, fontweight = 'bold')\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)\nax[1].set_xlabel('')\nax[1].set_xticklabels(ax[0].get_xticklabels(), rotation = 35, fontsize = 13, color = 'black')\nax[1].set_ylabel('not_nan_values', fontsize = 18, color ='black')\n\nfig,ax = plt.subplots(1, 1, figsize = (20, 8))\n\nbbox=[-0.2, 0, 1.2, 0.9]\nax.axis('off')\nax.title.set_text('')\nccolors = plt.cm.BuPu(np.full(len(train_stats.columns), 0.1))\n\nmpl_table = ax.table(cellText = train_stats.values, bbox=bbox, colLabels=train_stats.columns, colColours=ccolors)\nmpl_table.auto_set_font_size(False)\nmpl_table.auto_set_column_width(col=list(range(len(train_stats.columns))))\nmpl_table.set_fontsize(18)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:10:53.867991Z","iopub.execute_input":"2022-07-15T14:10:53.868282Z","iopub.status.idle":"2022-07-15T14:10:54.678011Z","shell.execute_reply.started":"2022-07-15T14:10:53.868255Z","shell.execute_reply":"2022-07-15T14:10:54.676852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(train.loc[(train.url_legal.notna()) & (train.license.isna())] ) == 0","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:10:54.679298Z","iopub.execute_input":"2022-07-15T14:10:54.679588Z","iopub.status.idle":"2022-07-15T14:10:54.686991Z","shell.execute_reply.started":"2022-07-15T14:10:54.679561Z","shell.execute_reply":"2022-07-15T14:10:54.685718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`url_legal` and `license` are always co-occurring. **They will be missing in the private test set.**","metadata":{}},{"cell_type":"code","source":"plt.style.use('ggplot')\nfig, ax = plt.subplots(2, 1, figsize = (20, 12))\n\nsns.histplot(data = train.loc[train.url_legal.notna()], x = 'target', \n ax = ax[0], kde=True, bins = 50,\n stat = 'density', color = 'red', edgecolor = 'black',\n alpha = 0.3, label = 'url_legal not NaN', \n linewidth = 3, line_kws= {'linewidth': 3})\n\nsns.histplot(data = train.loc[train.url_legal.isna()], x = 'target', \n ax = ax[0], kde=True, bins = 50, stat = 'density', \n alpha = 0.3, label = 'url_legal NaN', edgecolor = 'black',\n linewidth = 3, line_kws= {'linewidth': 3})\n\nax[0].legend(fontsize=18)\nax[0].set_xlabel('target', fontsize = 18)\nax[0].set_ylabel('Density', fontsize = 18)\nax[0].set_title('target distribution with url_legal NaN or not', fontsize = 15)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n\nsns.histplot(data = train.loc[train.url_legal.notna()], x = 'standard_error', \n ax = ax[1], kde=True, bins = 50,\n stat = 'density', color = 'red',\n alpha = 0.3, label = 'url_legal not NaN',\n edgecolor = 'black',\n linewidth = 3, line_kws= {'linewidth': 3})\n\nsns.histplot(data = train.loc[train.url_legal.isna()], x = 'standard_error', \n ax = ax[1], kde=True, bins = 50, stat = 'density', \n alpha = 0.3, label = 'url_legal NaN',\n edgecolor = 'black',\n linewidth = 3, line_kws= {'linewidth': 3})\n\nax[1].legend(fontsize=18)\nax[1].set_xlabel('standard_error', fontsize = 18)\nax[1].set_ylabel('Density', fontsize = 18)\nax[1].set_xlim(0.4, 0.7)\nax[1].set_title('standard_error distribution with url_legal NaN or not', \n fontsize = 15)\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)\n\nplt.subplots_adjust(hspace = 0.3)\nplt.suptitle(\"Distribution when url_legal is NaN or not\", fontsize = 20, fontweight = 'bold')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:20:27.997455Z","iopub.execute_input":"2022-07-15T14:20:27.998149Z","iopub.status.idle":"2022-07-15T14:20:29.061154Z","shell.execute_reply.started":"2022-07-15T14:20:27.998085Z","shell.execute_reply":"2022-07-15T14:20:29.060156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a little shift to the right in the `target` distribution when `url_legal` is not NaN.","metadata":{}},{"cell_type":"code","source":"plt.style.use('ggplot')\n\nfig, ax = plt.subplots(1, 2, figsize = (16, 6), gridspec_kw={'width_ratios': [1.5, 0.6]})\n\npercentiles_asked = [0.1, 0.25, 0.5, 0.75, 0.9]\npercentiles = train['target'].quantile(percentiles_asked).tolist()\n\nsns.histplot(data = train, x = 'target', ax = ax[0], kde=False, bins = 50,\n stat = 'density', \n alpha = 0.5, \n fill = True,\n linewidth = 3,\n edgecolor='black',\n color = 'red',\n #line_kws= {'linewidth': 5, 'color': 'red', 'alpha': 0.6}\n )\n\nsns.kdeplot(data = train, x = 'target', ax = ax[0], alpha = 0.01, fill = True, \n linewidth = 5, color = 'blue')\n\nfor m, percentile in enumerate(percentiles):\n ax[0].axvline(percentile, alpha = 0.35, ymin = 0, ymax = 1, linestyle = \":\", color = 'blue')\n ax[0].text(percentile-0.16, 0.43, \"{}\".format(percentiles_asked[m]), size = 12, alpha = 1)\n \nmean = train.target.mean().round(2)\nmedian = train.target.median().round(2)\nst_dev = train.target.std().round(2)\n\nax[0].text(-4.4, 0.4, \"mean: {}\".format(mean), size = 12, alpha = 1)\nax[0].text(-4.4, 0.36, \"median: {}\".format(median), size = 12, alpha = 1)\nax[0].text(-4.4, 0.32, \"std deviation: {}\".format(st_dev), size = 12, alpha = 1)\n\n#https://stackoverflow.com/questions/49926147/how-to-modify-edge-color-of-violinplot-using-seaborn/55131881 \n#per cambiare colore linea esterna\nsns.violinplot(y='target', data = train, ax=ax[1], inner = 'quartile',)\nfor l in ax[1].lines:\n l.set_linestyle('--')\n l.set_color('yellow')\n l.set_alpha(0.2)\n\nax[0].set_ylabel('Density', fontsize = 15)\nax[1].set_ylabel('target', fontsize = 15)\nax[0].set_xlabel('target', fontsize = 15)\nax[0].set_title('hist-kde plot', fontsize = 16)\nax[1].set_title('violin plot with quartiles', fontsize = 16)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n#plt.subplots_adjust(hspace = 0.8)\nfig.suptitle('Distribution of variable target', fontsize = 20, fontweight = 'bold')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:20:43.515072Z","iopub.execute_input":"2022-07-15T14:20:43.515477Z","iopub.status.idle":"2022-07-15T14:20:44.296135Z","shell.execute_reply.started":"2022-07-15T14:20:43.515446Z","shell.execute_reply":"2022-07-15T14:20:44.295022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('ggplot')\n\nfig, ax = plt.subplots(1, 2, figsize = (16, 6), gridspec_kw={'width_ratios': [1.5, 0.6]})\n\npercentiles_asked = [0.1, 0.25, 0.5, 0.75, 0.9]\npercentiles = train['standard_error'].quantile(percentiles_asked).tolist()\nprint(percentiles)\n\nsns.histplot(data = train, x = 'standard_error', ax = ax[0], kde=False, bins = 50,\n stat = 'density', \n alpha = 0.5, \n fill = True,\n linewidth = 3,\n edgecolor='black',\n color = 'red',\n )\n\nsns.kdeplot(data = train, x = 'standard_error', ax = ax[0], alpha = 0.01, fill = True, \n linewidth = 3, color = 'blue')\n\n#for m, percentile in enumerate(percentiles):\n# ax[0].axvline(percentile, alpha = 0.5, ymin = 0, ymax = 1, linestyle = \":\", color = 'blue')\n# ax[0].text(percentile-0.01, 16.5, \"{}\".format(percentiles_asked[m]), size = 12, alpha = 1)\n\nsns.violinplot(y='standard_error', data = train, ax=ax[1], inner = 'quartile')\nfor l in ax[1].lines:\n l.set_linestyle('--')\n l.set_color('yellow')\n l.set_alpha(0.3)\n \nmean = train.standard_error.mean().round(2)\nmedian = train.standard_error.median().round(2)\nst_dev = train.standard_error.std().round(2)\n\nax[0].text(0.0, 16, \"mean: {}\".format(mean), size = 12, alpha = 1)\nax[0].text(0.0, 14, \"median: {}\".format(median), size = 12, alpha = 1)\nax[0].text(0.0, 12, \"std deviation: {}\".format(st_dev), size = 12, alpha = 1)\n\nax[0].set_ylabel('Density', fontsize = 15)\nax[1].set_ylabel('standard_error', fontsize = 15)\nax[0].set_xlabel('standard_error', fontsize = 15)\nax[0].set_title('hist-kde plot', fontsize = 16)\nax[1].set_title('violin plot with quartiles', fontsize = 16)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n#ax[0].set_xlim(0.4, 0.7)\nax[0].set_ylim(0, 17)\nfig.suptitle('Distribution of variable standard_error', fontsize = 20, fontweight = 'bold')\nplt.subplots_adjust(hspace = 0.6)\n\nax[0].annotate('crazy outlier in standard_error', xy=(0.0, 0), xycoords='data',\n xytext=(0.4, 0.2), textcoords='axes fraction', fontsize = 14,\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='right', verticalalignment='top',\n )","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:20:56.028293Z","iopub.execute_input":"2022-07-15T14:20:56.028668Z","iopub.status.idle":"2022-07-15T14:20:56.556889Z","shell.execute_reply.started":"2022-07-15T14:20:56.028636Z","shell.execute_reply":"2022-07-15T14:20:56.555767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nst_sc = StandardScaler()\n# comparison between the 2 after standardizing\n\n# target,standard_error distributions including also violin plots\n# aggiungere text con percentili e violinplot\nplt.style.use('ggplot')\n\nfig, ax = plt.subplots(1, 1, figsize = (16, 6))#, gridspec_kw={'height_ratios': [0.6, 1.5]})\n\n#sns.histplot(x = train['standard_error'], bins = 20,\n# ax = ax[0], alpha = 0.25, fill = True, label = 'standard_error', \n# linewidth = 3, color = 'blue')\n#\n#sns.histplot(x = train['target'], bins = 10,\n# ax = ax[0], alpha = 0.25, fill = True, label = 'target', \n# linewidth = 3, color = 'red', common_norm = True)\n\nsns.kdeplot(x = st_sc.fit_transform(train[['standard_error']])[:, 0], \n ax = ax, alpha = 0.25, fill = True, label = 'standard_error_normalized', \n linewidth = 3, color = 'blue')\n\nsns.kdeplot(x = st_sc.fit_transform(train[['target']])[:, 0], \n ax = ax, alpha = 0.25, fill = True, label = 'target_normalized', \n linewidth = 3, color = 'red')\n\nax.legend(fontsize = 20, loc = 'upper left')\n\nax.annotate('crazy outlier in standard_error', xy=(-14, 0), xycoords='data',\n xytext=(0.3, 0.1), textcoords='axes fraction', fontsize = 14,\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='right', verticalalignment='top',\n )\n\nax.set_ylabel('Density', fontsize = 15)\nfig.suptitle('Distribution of target and standard_error after normalization', \n fontsize = 20, fontweight = 'bold')\n\nplt.subplots_adjust(hspace = 0.6)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:10:56.825337Z","iopub.execute_input":"2022-07-15T14:10:56.825669Z","iopub.status.idle":"2022-07-15T14:10:57.161171Z","shell.execute_reply.started":"2022-07-15T14:10:56.825640Z","shell.execute_reply":"2022-07-15T14:10:57.159984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# score based on number of letters/words\nfig, ax = plt.subplots(2, 1, figsize = (20, 14))\nsns.scatterplot(y = 'target', \n x = 'standard_error',\n data = train, \n ax = ax[0], \n color = 'blue', \n sizes = [5],\n alpha = 0.3)\n\nax[0].set_title(\"target vs standard_error\")\nax[0].set_ylabel('target', fontsize = 15)\nax[0].set_xlabel('standard_error', fontsize = 15)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n\nfig.suptitle(\"target vs standard_error\", fontsize = 20, fontweight = 'bold')\n\nax[0].set_ylim(-4, 2)\nax[0].set_xlim(0, 1)\n\nax[0].annotate('crazy outlier in standard_error', xy=(0, 0), xycoords='data',\n xytext=(0.4, 0.95), textcoords='axes fraction',fontsize = 14,\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='right', verticalalignment='top',\n )\n\nx = st_sc.fit_transform(train[['standard_error']])[:, 0]\ny = st_sc.fit_transform(train[['target']])[:, 0]\n\nsns.scatterplot(y = y, \n x = x,\n ax = ax[1], \n color = 'blue', \n sizes = [5],\n alpha = 0.3)\n\nax[1].set_title(\"target vs standard_error after normalization\")\nax[1].set_ylabel('target', fontsize = 15)\nax[1].set_xlabel('standard_error', fontsize = 15)\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:10:57.162564Z","iopub.execute_input":"2022-07-15T14:10:57.162908Z","iopub.status.idle":"2022-07-15T14:10:57.697997Z","shell.execute_reply.started":"2022-07-15T14:10:57.162847Z","shell.execute_reply":"2022-07-15T14:10:57.696847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Higher standard error for more extreme values of target**","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nlr = LinearRegression(fit_intercept = True)\n\nlr.fit(y = train['target'], X = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len().to_frame())\nfitted = lr.predict(X = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len().to_frame())\n\nfig, ax = plt.subplots(1, 1, figsize = (16, 8))\nsns.scatterplot(y = train['target'], x = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\", regex=True).str.len(),\n ax = ax, color= 'blue', sizes= [5], alpha= 0.3)\n\nsns.lineplot(x = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\", regex = True).str.len(),\n y = fitted, ax = ax, linewidth = 5, color = 'red')\n\nax.set_ylabel('target', fontsize = 15)\nax.set_xlabel('number of letters in excerpt', fontsize = 15)\nax.set_title('Negative correlation between number of letters in excerpt and target', fontsize = 13)\nax.tick_params(axis='both', which='major', labelsize=14)\nax.tick_params(axis='both', which='minor', labelsize=14)\n\nfig.suptitle(\"Number of letters in excerpt vs target (fitted with Sklearn)\", fontsize = 20, fontweight = 'bold')\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:10:57.699304Z","iopub.execute_input":"2022-07-15T14:10:57.699602Z","iopub.status.idle":"2022-07-15T14:11:11.082291Z","shell.execute_reply.started":"2022-07-15T14:10:57.699573Z","shell.execute_reply":"2022-07-15T14:11:11.081238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 2, figsize = (16, 16))\n\nax = axes.ravel()\n\nsns.regplot(y = train['target'], x = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len(),\n ax = ax[0], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, \n line_kws = {'color': 'red', 'linewidth': 3, 'alpha': 0.5})\n\nsns.regplot(y = train['standard_error'], x = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len(),\n ax = ax[1], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, \n line_kws = {'color': 'red', 'linewidth': 3, 'alpha': 0.5})\n\n\n\nlr = LinearRegression()\n\n\nax[0].set_ylabel('target', fontsize = 15)\nax[0].set_xlabel('Number of letters in Excerpt', fontsize = 15)\nax[0].set_title('Number of letters in excerpt vs target', fontsize = 16)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n\nax[1].set_ylabel('standard_error', fontsize = 15)\nax[1].set_xlabel('Number of letters in Excerpt', fontsize = 15)\nax[1].set_title('Number of letters in excerpt vs standard_error', fontsize = 16)\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)\nax[1].set_ylim(0.425, 0.65)\n\n\nsns.regplot(y = train['target'], x = train['excerpt'].str.split().str.len(),\n ax = ax[2], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\nsns.regplot(y = train['standard_error'], x = train['excerpt'].str.split().str.len(),\n ax = ax[3], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\n\nax[2].set_ylabel('target', fontsize = 15)\nax[2].set_xlabel('Number of words in Excerpt', fontsize = 15)\nax[2].set_title('Number of words in excerpt vs target', fontsize = 16)\nax[2].tick_params(axis='both', which='major', labelsize=14)\nax[2].tick_params(axis='both', which='minor', labelsize=14)\n\nax[3].set_ylabel('standard_error', fontsize = 15)\nax[3].set_xlabel('Number of words in Excerpt', fontsize = 15)\nax[3].set_title('Number of words in excerpt vs standard_error', fontsize = 16)\nax[3].tick_params(axis='both', which='major', labelsize=14)\nax[3].tick_params(axis='both', which='minor', labelsize=14)\nax[3].set_ylim(0.425, 0.65)\n\nsns.regplot(y = train['target'], x = train['excerpt'].str.split(pat='[.!?]+').str.len(),\n ax = ax[4], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\nsns.regplot(y = train['standard_error'], x = train['excerpt'].str.split(pat='[.!?]+').str.len(),\n ax = ax[5], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\n\nax[4].set_ylabel('target', fontsize = 15)\nax[4].set_xlabel('Number of sentences in Excerpt', fontsize = 15)\nax[4].set_title('Number of sentences in excerpt vs target', fontsize = 16)\nax[4].tick_params(axis='both', which='major', labelsize=14)\nax[4].tick_params(axis='both', which='minor', labelsize=14)\n\nax[5].set_ylabel('standard_error', fontsize = 15)\nax[5].set_xlabel('Number of sentences in Excerpt', fontsize = 15)\nax[5].set_title('Number of sentences in excerpt vs standard_error', fontsize = 16)\nax[5].tick_params(axis='both', which='major', labelsize=14)\nax[5].tick_params(axis='both', which='minor', labelsize=14)\nax[5].set_ylim(0.425, 0.65)\n\nplt.subplots_adjust(hspace = 0.3)\n\nfig.suptitle(\"Number of letters/words/sentences in excerpt vs target/standard_error (using sns.regplot)\", \n fontsize = 20, fontweight = 'bold')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:11.083652Z","iopub.execute_input":"2022-07-15T14:11:11.083980Z","iopub.status.idle":"2022-07-15T14:11:14.774785Z","shell.execute_reply.started":"2022-07-15T14:11:11.083949Z","shell.execute_reply":"2022-07-15T14:11:14.773661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df = train.copy()\n\ncorr_df['n_letters_excerpt'] = corr_df['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len()\ncorr_df['n_words_excerpt'] = corr_df['excerpt'].str.split().str.len()\ncorr_df['n_sentences_excerpt'] = corr_df['excerpt'].str.split(pat = '[.!?]+').str.len()\n\ncorr_df = corr_df[['target', 'standard_error', 'n_letters_excerpt', 'n_words_excerpt', 'n_sentences_excerpt']]\n\ncorr_matrix = round(corr_df.corr(), 2)\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n \nfig, ax = plt.subplots(1, 1, figsize = (10, 10))\ncolors = sns.color_palette('rocket', 21)\nlevels = np.linspace(-1, 1, 21)\ncmap_plot, norm = matplotlib.colors.from_levels_and_colors(levels, colors, extend=\"max\")\nsns.heatmap(corr_matrix, mask=mask, annot=True, ax = ax, \n cmap = cmap_plot, norm = norm, annot_kws={\"size\": 15, \"color\": 'black'})\nax.hlines([0, 1, 2, 3, 4], *ax.get_xlim(), color = 'black')\nax.vlines([0, 1, 2, 3, 4], *ax.get_ylim(), color = 'black')\nax.xaxis.set_ticks_position('bottom')\nax.set_title('Distinct values for each variable', fontsize = 20)\nax.tick_params(axis='both', which='major', labelsize=14)\nax.tick_params(axis='both', which='minor', labelsize=14)\nax.set_xticklabels(ax.get_xticklabels(), rotation = 35, fontsize = 15, color = 'black')\nax.set_yticklabels(ax.get_yticklabels(), rotation = 35, fontsize = 15, color = 'black')\nax.xaxis.label.set_size(14)\n\ncircle_rad = 25 # This is the radius, in points\nax.plot(0.5, 2.5, 'o',\n ms=circle_rad * 2, mec='w', mfc='none', mew=4)\n\nfig.suptitle('Correlation Matrix for {}'.format('train.csv'), \n fontsize = 20, color = 'black', fontweight = 'bold')\nplt.title(\"Just to sum up: as seen before there's a negative correlation between number of letters and target\", fontsize = 12)\nfig.show()\n ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:14.776210Z","iopub.execute_input":"2022-07-15T14:11:14.776507Z","iopub.status.idle":"2022-07-15T14:11:15.595638Z","shell.execute_reply.started":"2022-07-15T14:11:14.776478Z","shell.execute_reply":"2022-07-15T14:11:15.594296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df['average_word_length'] = (corr_df['n_letters_excerpt']/corr_df['n_words_excerpt']).round(3)\ncorr_df['average_sentence_length'] = (corr_df['n_sentences_excerpt']/corr_df['n_words_excerpt']).round(3)\n\ncorr_matrix = round(corr_df.corr(), 2)\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n \nfig, ax = plt.subplots(1, 1, figsize = (8, 8))\ncolors = sns.color_palette('rocket', 21)\nlevels = np.linspace(-1, 1, 21)\ncmap_plot, norm = matplotlib.colors.from_levels_and_colors(levels, colors, extend=\"max\")\nsns.heatmap(corr_matrix, mask=mask, annot=True, ax = ax, \n cmap = cmap_plot, norm = norm, annot_kws={\"size\": 15, \"color\": 'black'})\nax.hlines([0, 1, 2, 3, 4, 5, 6], *ax.get_xlim(), color = 'black')\nax.vlines([0, 1, 2, 3, 4, 5, 6], *ax.get_ylim(), color = 'black')\nax.xaxis.set_ticks_position('bottom')\nax.set_title('Distinct values for each variable', fontsize = 20)\nax.tick_params(axis='both', which='major', labelsize=14)\nax.tick_params(axis='both', which='minor', labelsize=14)\nax.set_xticklabels(ax.get_xticklabels(), rotation = 45, fontsize = 12, color = 'black')\nax.set_yticklabels(ax.get_yticklabels(), rotation = 35, fontsize = 15, color = 'black')\nax.xaxis.label.set_size(14)\n\ncircle_rad = 25 # This is the radius, in points\nax.plot(0.5, 5.5, 'o',\n ms=circle_rad * 2, mec='w', mfc='none', mew=4)\n\nfig.suptitle('Correlation Matrix for {}, after creating average_word_length'.format('train.csv'), \n fontsize = 20, color = 'black', fontweight = 'bold')\nplt.title(\"An even more negative correlation between average_length and target\", fontsize = 12)\nfig.show()\n ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:15.597157Z","iopub.execute_input":"2022-07-15T14:11:15.597492Z","iopub.status.idle":"2022-07-15T14:11:16.058547Z","shell.execute_reply.started":"2022-07-15T14:11:15.597460Z","shell.execute_reply":"2022-07-15T14:11:16.057263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I expect more difficult scores when the number of rare words is higher... ","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer \ncvect = CountVectorizer(binary = True)\nbag_of_words = cvect.fit_transform(train.excerpt)\nprint(\"Total Number of words (no stemming nor lemmatization): {}\".format(bag_of_words.shape[1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:16.060203Z","iopub.execute_input":"2022-07-15T14:11:16.060616Z","iopub.status.idle":"2022-07-15T14:11:16.725330Z","shell.execute_reply.started":"2022-07-15T14:11:16.060572Z","shell.execute_reply":"2022-07-15T14:11:16.724180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = bag_of_words.sum(axis = 0)\ndf_counts = pd.DataFrame({'counts': np.squeeze(np.asarray(counts)), 'word': cvect.get_feature_names()})\ndisplay(df_counts.sort_values('counts', ascending = False, ignore_index = True))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:16.726614Z","iopub.execute_input":"2022-07-15T14:11:16.726951Z","iopub.status.idle":"2022-07-15T14:11:16.774467Z","shell.execute_reply.started":"2022-07-15T14:11:16.726919Z","shell.execute_reply":"2022-07-15T14:11:16.773162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I choose the 10% most rare words","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:25:45.200635Z","iopub.execute_input":"2021-05-23T16:25:45.200949Z","iopub.status.idle":"2021-05-23T16:25:45.206304Z","shell.execute_reply.started":"2021-05-23T16:25:45.20092Z","shell.execute_reply":"2021-05-23T16:25:45.205697Z"}}},{"cell_type":"code","source":"most_rare = df_counts.iloc[int(0.9*len(df_counts)):]['word'].tolist()\nprint(\"Some 'rare' words: \\n\")\nnp.random.choice(most_rare, 5).tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:16.776038Z","iopub.execute_input":"2022-07-15T14:11:16.776380Z","iopub.status.idle":"2022-07-15T14:11:16.787322Z","shell.execute_reply.started":"2022-07-15T14:11:16.776348Z","shell.execute_reply":"2022-07-15T14:11:16.786281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# score based on number of letters/words\n\ndef how_many_rare(excerpt, most_rare, list_like = True):\n if list_like:\n #list_like may take some time (< 1 minute)\n return len([i for i in excerpt.split() if i in most_rare])\n else:\n return len(set(excerpt.split()).intersection(set(most_rare)))\n\ntrain['rare_words'] = train.apply(lambda x: how_many_rare(x.excerpt, most_rare), 1)\ntrain['rare_words_over_nwords'] = train['rare_words']/train.excerpt.str.split().apply(lambda x: len(x))\n\n\nfig, ax = plt.subplots(2, 1, figsize = (12, 12))\nsns.regplot(y = train['target'], x = train['rare_words'],\n ax = ax[0], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\nax[0].set_ylabel('target', fontsize = 15)\nax[0].set_xlabel('', fontsize = 15)\nax[0].set_title('Number of rare words in excerpt vs target', fontsize = 16)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n\nsns.regplot(y = train['standard_error'], x = train['rare_words'],\n ax = ax[1], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\nax[1].set_ylabel('standard_error', fontsize = 15)\nax[1].set_xlabel('', fontsize = 15)\nax[1].set_title('Number of rare words in excerpt vs standard_error', fontsize = 16)\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)\nax[1].set_ylim(0.4, 0.7)\n\nfig.suptitle(\"Number of 'rare words' vs target\", fontsize = 20, fontweight = 'bold')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:16.788989Z","iopub.execute_input":"2022-07-15T14:11:16.789688Z","iopub.status.idle":"2022-07-15T14:11:36.510189Z","shell.execute_reply.started":"2022-07-15T14:11:16.789640Z","shell.execute_reply":"2022-07-15T14:11:36.509034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n
test.csv
","metadata":{"execution":{"iopub.status.busy":"2021-05-27T06:29:49.777323Z","iopub.execute_input":"2021-05-27T06:29:49.777731Z","iopub.status.idle":"2021-05-27T06:29:49.785379Z","shell.execute_reply.started":"2021-05-27T06:29:49.777694Z","shell.execute_reply":"2021-05-27T06:29:49.78327Z"}}},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:36.511453Z","iopub.execute_input":"2022-07-15T14:11:36.511725Z","iopub.status.idle":"2022-07-15T14:11:36.525426Z","shell.execute_reply.started":"2022-07-15T14:11:36.511700Z","shell.execute_reply":"2022-07-15T14:11:36.524400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['n_letters_excerpt'] = test['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len()\ntest['n_words_excerpt'] = test['excerpt'].str.split().str.len()\ntest['n_sentences_excerpt'] = test['excerpt'].str.split(pat = '[.!?]+').str.len()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:36.526747Z","iopub.execute_input":"2022-07-15T14:11:36.527098Z","iopub.status.idle":"2022-07-15T14:11:36.540728Z","shell.execute_reply.started":"2022-07-15T14:11:36.527067Z","shell.execute_reply":"2022-07-15T14:11:36.539591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:36.542232Z","iopub.execute_input":"2022-07-15T14:11:36.542853Z","iopub.status.idle":"2022-07-15T14:11:36.567251Z","shell.execute_reply.started":"2022-07-15T14:11:36.542820Z","shell.execute_reply":"2022-07-15T14:11:36.566337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target,standard_error distributions including also violin plots\n# aggiungere text con percentili e violinplot\nplt.style.use('ggplot')\n\nfig, ax = plt.subplots(1, 1, figsize = (16, 6))\n\npercentiles_asked = [0.1, 0.25, 0.5, 0.75, 0.9]\npercentiles = corr_df['n_letters_excerpt'].quantile(percentiles_asked).tolist()\n\nsns.histplot(data = corr_df, x = 'n_letters_excerpt', ax = ax, kde=False, bins = 50,\n stat = 'density', \n alpha = 0.2, \n fill = True,\n linewidth = 3,\n edgecolor='black',\n color = 'red',\n )\n\nsns.kdeplot(data =corr_df, x = 'n_letters_excerpt', ax = ax, alpha = 0.01, fill = True, \n linewidth = 3, color = 'blue')\n\nfor idx, row in test.sort_values('n_letters_excerpt', ignore_index=True).iterrows():\n \n ax.vlines(x = row['n_letters_excerpt'], ymin = 0, ymax = 0.0045, colors = 'red', linewidth = 2, label = row.id,\n linestyles = 'dashed')\n if idx <2:\n ax.text(x = row['n_letters_excerpt']-40, y = 0.0047, s=row.id, fontsize = 9)\n if idx >4:\n ax.text(x = row['n_letters_excerpt'], y = 0.0047, s=row.id, fontsize = 9)\n if idx == 2:\n ax.text(x = row['n_letters_excerpt']-40, y = 0.0046, s=row.id, fontsize = 9)\n if idx == 3:\n ax.text(x = row['n_letters_excerpt']-30, y = 0.00475, s=row.id, fontsize = 9)\n if idx == 4:\n ax.text(x = row['n_letters_excerpt']-30, y = 0.0046, s=row.id, fontsize = 9)\n \n\nax.set_ylabel('Density', fontsize = 15)\nax.set_xlabel('n_letters_excerpt', fontsize = 15)\nax.set_ylim(0.0, 0.005)\nax.set_title('hist-kde plot', fontsize = 16)\nax.tick_params(axis='both', which='major', labelsize=14)\nax.tick_params(axis='both', which='minor', labelsize=14)\n\nfig.suptitle('Distribution of number of letters in excerpt: train vs test with corresponding ids', fontsize = 20, fontweight = 'bold')\nplt.subplots_adjust(hspace = 0.6)\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:36.568702Z","iopub.execute_input":"2022-07-15T14:11:36.569109Z","iopub.status.idle":"2022-07-15T14:11:37.012455Z","shell.execute_reply.started":"2022-07-15T14:11:36.569075Z","shell.execute_reply":"2022-07-15T14:11:37.011238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target,standard_error distributions including also violin plots\n# aggiungere text con percentili e violinplot\nplt.style.use('ggplot')\n\nfig, ax = plt.subplots(1, 1, figsize = (16, 6))\n\npercentiles_asked = [0.1, 0.25, 0.5, 0.75, 0.9]\npercentiles = corr_df['n_words_excerpt'].quantile(percentiles_asked).tolist()\n\nsns.histplot(data = corr_df, x = 'n_words_excerpt', ax = ax, kde=False, bins = 25,\n stat = 'density', \n alpha = 0.2, \n fill = True,\n linewidth = 3,\n edgecolor='black',\n color = 'red',\n )\n\nsns.kdeplot(data =corr_df, x = 'n_words_excerpt', ax = ax, alpha = 0.01, fill = True, \n linewidth = 3, color = 'blue')\n\nfor idx, row in test.sort_values('n_words_excerpt', ignore_index=True).iterrows():\n \n ax.vlines(x = row['n_words_excerpt'], ymin = 0, ymax = 0.025, colors = 'red', linewidth = 2, label = row.id,\n linestyles = 'dashed')\n \n if idx == 0:\n ax.text(x = row['n_words_excerpt']-5, y = 0.026, s=row.id, fontsize = 9)\n if idx == 1:\n ax.text(x = row['n_words_excerpt']-2, y = 0.028, s=row.id, fontsize = 9)\n if idx == 5:\n ax.text(x = row['n_words_excerpt']-3, y = 0.026, s=row.id, fontsize = 9)\n if idx == 6:\n ax.text(x = row['n_words_excerpt']-1, y = 0.028, s=row.id, fontsize = 9)\n if (idx>=2) & (idx<=4):\n ax.text(x = row['n_words_excerpt']-3, y = 0.026, s=row.id, fontsize = 9)\n\nax.set_ylabel('Density', fontsize = 15)\nax.set_xlabel('n_words_excerpt', fontsize = 15)\nax.set_ylim(0.0, 0.03)\nax.set_title('hist-kde plot', fontsize = 16)\nax.tick_params(axis='both', which='major', labelsize=14)\nax.tick_params(axis='both', which='minor', labelsize=14)\n\nfig.suptitle('Distribution of number of words in excerpt: train vs test with corresponding ids', fontsize = 20, fontweight = 'bold')\nplt.subplots_adjust(hspace = 0.6)\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-15T14:11:37.016036Z","iopub.execute_input":"2022-07-15T14:11:37.016402Z","iopub.status.idle":"2022-07-15T14:11:37.354554Z","shell.execute_reply.started":"2022-07-15T14:11:37.016371Z","shell.execute_reply":"2022-07-15T14:11:37.353325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sentence'] = train.excerpt.str.split(pat = '[.!?]+')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:37.356606Z","iopub.execute_input":"2022-07-15T14:11:37.357054Z","iopub.status.idle":"2022-07-15T14:11:37.477532Z","shell.execute_reply.started":"2022-07-15T14:11:37.357009Z","shell.execute_reply":"2022-07-15T14:11:37.476278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_sentences(x: list):\n tags = []\n for sentence in x:\n pos_tag = [i[1] for i in nltk.pos_tag(nltk.word_tokenize(sentence), tagset= 'universal')]\n tags+=pos_tag\n return tags\n\ndef tokenize_sentences(x: list):\n tags = []\n for sentence in x:\n pos_tag = [token.pos_ for token in nlp(sentence)]\n tags+=pos_tag\n return tags","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:37.479224Z","iopub.execute_input":"2022-07-15T14:11:37.479677Z","iopub.status.idle":"2022-07-15T14:11:37.487388Z","shell.execute_reply.started":"2022-07-15T14:11:37.479628Z","shell.execute_reply":"2022-07-15T14:11:37.485972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\ntrain['pos_tag'] = train.sentence.apply(lambda x: tokenize_sentences(x))\nprint(\"Time Elapsed: {}\".format(time.time()-start_time))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:11:37.488911Z","iopub.execute_input":"2022-07-15T14:11:37.489436Z","iopub.status.idle":"2022-07-15T14:16:22.893459Z","shell.execute_reply.started":"2022-07-15T14:11:37.489392Z","shell.execute_reply":"2022-07-15T14:16:22.892448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['number_of_verbs'] = train['pos_tag'].apply(lambda x: len([i for i in x if i == 'VERB'])/len(x))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:16:22.895259Z","iopub.execute_input":"2022-07-15T14:16:22.895575Z","iopub.status.idle":"2022-07-15T14:16:22.932355Z","shell.execute_reply.started":"2022-07-15T14:16:22.895541Z","shell.execute_reply":"2022-07-15T14:16:22.931101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# score based on number of letters/words\n\nfig, ax = plt.subplots(1, 2, figsize = (16, 12), gridspec_kw={'width_ratios': [2, 1]})\nsns.regplot(y = train['target'], x = train['number_of_verbs'],\n ax = ax[0], scatter_kws = {'color': 'blue', 'sizes': [5], 'alpha': 0.3}, line_kws = {'color': 'red', \n 'linewidth': 3, 'alpha': 0.5})\n\nax[0].set_ylabel('target', fontsize = 15)\nax[0].set_xlabel('', fontsize = 15)\nax[0].set_title('Percentage of verbs in excerpt vs target', fontsize = 16)\nax[0].tick_params(axis='both', which='major', labelsize=14)\nax[0].tick_params(axis='both', which='minor', labelsize=14)\n\nfig.suptitle(\"Percentage of verbs vs target/standard_error\", fontsize = 20, fontweight = 'bold')\nplt.title(\"More verbs seem to make the comprehension easier\", fontsize = 12, fontweight = 'bold')\n\ncorr_df = train.copy()\n\ncorr_df = corr_df[['target', 'standard_error', 'number_of_verbs']]\n\ncorr_matrix = round(corr_df.corr(), 2)\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n \ncolors = sns.color_palette('rocket', 21)\nlevels = np.linspace(-1, 1, 21)\ncmap_plot, norm = matplotlib.colors.from_levels_and_colors(levels, colors, extend=\"max\")\nsns.heatmap(corr_matrix, mask=mask, annot=True, ax = ax[1], \n cmap = cmap_plot, norm = norm, annot_kws={\"size\": 15, \"color\": 'black'})\nax[1].hlines([0, 1, 2, 3, 4, 5], *ax[1].get_xlim(), color = 'black')\nax[1].vlines([0, 1, 2, 3, 4, 5], *ax[1].get_ylim(), color = 'black')\nax[1].xaxis.set_ticks_position('bottom')\nax[1].set_title('Correlation matrix', fontsize = 20)\nax[1].tick_params(axis='both', which='major', labelsize=14)\nax[1].tick_params(axis='both', which='minor', labelsize=14)\nax[1].set_xticklabels(ax[1].get_xticklabels(), rotation = 35, fontsize = 15, color = 'black')\nax[1].set_yticklabels(ax[1].get_yticklabels(), rotation = 35, fontsize = 15, color = 'black')\nax[1].xaxis.label.set_size(14)\n\ncircle_rad = 25 # This is the radius, in points\nax[1].plot(0.5, 2.5, 'o',\n ms=circle_rad * 2, mec='w', mfc='none', mew=4)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:16:22.934083Z","iopub.execute_input":"2022-07-15T14:16:22.934452Z","iopub.status.idle":"2022-07-15T14:16:23.781541Z","shell.execute_reply.started":"2022-07-15T14:16:22.934404Z","shell.execute_reply":"2022-07-15T14:16:23.780471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['n_letters_excerpt'] = train['excerpt'].str.replace(\"[^a-zA-Z0-9-]\", \"\").str.len()\ntrain['n_words_excerpt'] = train['excerpt'].str.split().str.len()\ntrain['n_sentences_excerpt'] = train['excerpt'].str.split(pat = '[.!?]+').str.len()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:16:23.782962Z","iopub.execute_input":"2022-07-15T14:16:23.783270Z","iopub.status.idle":"2022-07-15T14:16:24.226210Z","shell.execute_reply.started":"2022-07-15T14:16:23.783241Z","shell.execute_reply":"2022-07-15T14:16:24.225166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"?px.scatter_3d","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:16:24.227647Z","iopub.execute_input":"2022-07-15T14:16:24.228077Z","iopub.status.idle":"2022-07-15T14:16:24.234809Z","shell.execute_reply.started":"2022-07-15T14:16:24.228035Z","shell.execute_reply":"2022-07-15T14:16:24.233612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfig = px.scatter_3d(train.assign(size=2), x='n_words_excerpt', y='number_of_verbs', z='target', size = 'size', color = 'target', opacity=0.7)\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=0))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T14:16:24.236326Z","iopub.execute_input":"2022-07-15T14:16:24.236611Z","iopub.status.idle":"2022-07-15T14:16:26.907787Z","shell.execute_reply.started":"2022-07-15T14:16:24.236585Z","shell.execute_reply":"2022-07-15T14:16:26.906411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And this is it :)","metadata":{}}]}