{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on July 20, 2023. By Marília Prata, mpwolke.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-21T00:24:19.371837Z","iopub.execute_input":"2023-07-21T00:24:19.372171Z","iopub.status.idle":"2023-07-21T00:36:40.471293Z","shell.execute_reply.started":"2023-07-21T00:24:19.372144Z","shell.execute_reply":"2023-07-21T00:36:40.470487Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Kazi Nazrul\n\n\"Kazi Nazrul Islam (Bengali: কাজী নজরুল ইসলাম; 26 May 1899 – 29 August 1976) was an Indian and later Bangladeshi poet, writer, musician, and is the national poet of Bangladesh. Nazrul is regarded as one of the greatest poets in Bengali literature.\"\n\n\"Popularly known as Nazrul, he produced a large body of poetry, music, messages, novels, stories, etc. with themes that included equality, justice, anti-imperialism, humanity, rebellion against oppression and religious devotion. Nazrul's activism for political and social justice as well as writing a poem titled as \"Bidrohī\", meaning \"the rebel\" in Bengali, earned him the title of \"Bidrohī Kôbi\" (Rebel Poet). His compositions form the avant-garde music genre of Nazrul Gīti (Music of Nazrul).\"\n\nhttps://en.wikipedia.org/wiki/Kazi_Nazrul_Islam","metadata":{}},{"cell_type":"markdown","source":"![](https://i.ytimg.com/vi/EWivqMNMEU4/maxresdefault.jpg)","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/bengaliai-speech/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:37:51.806140Z","iopub.execute_input":"2023-07-21T00:37:51.806557Z","iopub.status.idle":"2023-07-21T00:37:55.678481Z","shell.execute_reply.started":"2023-07-21T00:37:51.806523Z","shell.execute_reply":"2023-07-21T00:37:55.677443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/bengaliai-speech/sample_submission.csv\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:39:23.545418Z","iopub.execute_input":"2023-07-21T00:39:23.546565Z","iopub.status.idle":"2023-07-21T00:39:23.560101Z","shell.execute_reply.started":"2023-07-21T00:39:23.546516Z","shell.execute_reply":"2023-07-21T00:39:23.559107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#History of Kazi Nazrul Islam\n\n\n\nhttps://www.youtube.com/watch?v=EWivqMNMEU4","metadata":{}},{"cell_type":"code","source":"# Other \nimport librosa\nimport librosa.display\nimport json\nimport tensorflow as tf\nfrom matplotlib.pyplot import specgram\nimport glob \nimport os\nfrom tqdm import tqdm\nimport pickle\nimport IPython.display as ipd # To play sound in the notebook","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:21:15.458804Z","iopub.execute_input":"2023-07-20T23:21:15.459191Z","iopub.status.idle":"2023-07-20T23:21:23.428080Z","shell.execute_reply.started":"2023-07-20T23:21:15.459162Z","shell.execute_reply":"2023-07-20T23:21:23.426927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Poem Recital by Kasi Nazrul Islam?","metadata":{}},{"cell_type":"code","source":"#By Eu Jin Lok https://www.kaggle.com/ejlok1/audio-emotion-part-5-data-augmentation/notebook\n\n# Use one audio file in previous parts again\nfname = '/kaggle/input/bengaliai-speech/examples/Poem Recital.wav' \ndata, sampling_rate = librosa.load(fname)\nplt.figure(figsize=(15, 5))\nlibrosa.display.waveshow(data, sr=sampling_rate)#librosa.display' has no attribute 'waveplot'\n\n# Paly it again to refresh our memory\nipd.Audio(data, rate=sampling_rate)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:21:42.376223Z","iopub.execute_input":"2023-07-20T23:21:42.377565Z","iopub.status.idle":"2023-07-20T23:21:54.438746Z","shell.execute_reply.started":"2023-07-20T23:21:42.377519Z","shell.execute_reply":"2023-07-20T23:21:54.437741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Naari (Woman) by Kasi Nazrul\n\n\"I don't see any difference\" - আমি কোন পার্থক্য দেখতে না\n\n\"Between a man and woman\" - একজন পুরুষ এবং মহিলার মধ্যে\n\n\"Whatever great or benevolent achievements\" - যাই হোক না কেন মহান বা পরোপকারী অর্জন\n\n\"That are in this world\" - যে এই পৃথিবীতে আছে\n\n\"Half of that was by woman,\" - এর অর্ধেক ছিল নারীর দ্বারা,\n\n\"The other half by man.\" - বাকি অর্ধেক মানুষের দ্বারা।\n\n\"Āmi kōna pārthakya dēkhatē nā\nēkajana puruṣa ēbaṁ mahilāra madhyē\nyā'i hōka nā kēna mahāna bā parōpakārī arjana\"\n\nhttps://translate.google.com/?sl=en&tl=bn&text=I%20don%27t%20see%20any%20difference%0ABetween%20a%20man%20and%20woman%0AWhatever%20great%20or%20benevolent%20achievements%0AThat%20are%20in%20this%20world%0AHalf%20of%20that%20was%20by%20woman%2C%0AThe%20other%20half%20by%20man.&op=translate","metadata":{}},{"cell_type":"code","source":"sentence = \"\"\"আমি কোন পার্থক্য দেখতে না\nএকজন পুরুষ এবং মহিলার মধ্যে\nযাই হোক না কেন মহান বা পরোপকারী অর্জন\nযে এই পৃথিবীতে আছে\nএর অর্ধেক ছিল নারীর দ্বারা,\nবাকি অর্ধেক মানুষের দ্বারা।\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:41:06.914176Z","iopub.execute_input":"2023-07-20T23:41:06.915036Z","iopub.status.idle":"2023-07-20T23:41:06.920124Z","shell.execute_reply.started":"2023-07-20T23:41:06.914995Z","shell.execute_reply":"2023-07-20T23:41:06.919028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget -c https://raw.githubusercontent.com/hmoazzem/bangla-fonts/master/kalpurush.ttf","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-21T00:39:52.342765Z","iopub.execute_input":"2023-07-21T00:39:52.343086Z","iopub.status.idle":"2023-07-21T00:39:52.918792Z","shell.execute_reply.started":"2023-07-21T00:39:52.343062Z","shell.execute_reply":"2023-07-21T00:39:52.917757Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Bidrohi (The Rebel) By Kasi Nazrul\n\nI am the unutterable grief, - আমি অবর্ণনীয় দুঃখ,\n\nI am the trembling first touch of the virgin, - আমি কুমারীর কাঁপা প্রথম স্পর্শ,\n\nI am the throbbing tenderness of her first stolen kiss. আমি তার প্রথম চুরি চুম্বনের স্পন্দিত কোমলতা.\n\nI am the fleeting glance of the veiled beloved, আমি অবগুণ্ঠিত প্রিয়তমের ক্ষণস্থায়ী দৃষ্টি,\n\nI am her constant surreptitious gaze...আমি তার অবিরাম গোপন দৃষ্টি...\n\nI am the burning volcano in the bosom of the earth,আমি পৃথিবীর বুকে জ্বলন্ত আগ্নেয়গিরি,\n\nI am the wildfire of the woods, - আমি বনের দাবানল,\n\nI am Hell's mad terrific sea of wrath! - আমি নরকের ক্রোধের ভয়ঙ্কর সাগর!\n\nI ride on the wings of lightning with joy and profundity,আমি আনন্দ এবং গভীরতা নিয়ে বজ্রের ডানায় চড়েছি,\n\nI scatter misery and fear all around, - আমি চারিদিকে দুঃখ এবং ভয় ছড়িয়ে দিই,\n\nI bring earthquakes on this world! \"(8th stanza)\" আমি এই পৃথিবীতে ভূমিকম্প আনি! \"(অষ্টম স্তবক)\"\n\nI am the rebel eternal, - আমি চির বিদ্রোহী,\n\nI raise my head beyond this world, - আমি এই পৃথিবীর ওপারে মাথা তুলেছি,\n\nHigh, ever erect and alone! - উচ্চ, কখনও খাড়া এবং একা!\n\nhttps://translate.google.com/?","metadata":{}},{"cell_type":"code","source":"text = \"\"\"আমি অবর্ণনীয় দুঃখ,\nআমি কুমারীর কাঁপা প্রথম স্পর্শ,\nআমি তার প্রথম চুরি চুম্বনের স্পন্দিত কোমলতা.\nআমি অবগুণ্ঠিত প্রিয়তমের ক্ষণস্থায়ী দৃষ্টি,\nআমি তার অবিরাম গোপন দৃষ্টি...\n\nআমি পৃথিবীর বুকে জ্বলন্ত আগ্নেয়গিরি,\nআমি বনের দাবানল,\nআমি নরকের ক্রোধের ভয়ঙ্কর সাগর!\nআমি আনন্দ এবং গভীরতা নিয়ে বজ্রের ডানায় চড়েছি,\nআমি চারিদিকে দুঃখ এবং ভয় ছড়িয়ে দিই,\nআমি এই পৃথিবীতে ভূমিকম্প আনি! \"(অষ্টম স্তবক)\"\n\nআমি চির বিদ্রোহী,\nআমি এই পৃথিবীর ওপারে মাথা তুলেছি,\nউচ্চ, কখনও খাড়া এবং একা!\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:55:57.202117Z","iopub.execute_input":"2023-07-20T23:55:57.202989Z","iopub.status.idle":"2023-07-20T23:55:57.208765Z","shell.execute_reply.started":"2023-07-20T23:55:57.202948Z","shell.execute_reply":"2023-07-20T23:55:57.207653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by https://infovis.fh-potsdam.de/tutorials/infovis5text.html\n\nimport nltk\n\nwords = nltk.word_tokenize(text)\nwords","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:56:35.253208Z","iopub.execute_input":"2023-07-20T23:56:35.253617Z","iopub.status.idle":"2023-07-20T23:56:35.264561Z","shell.execute_reply.started":"2023-07-20T23:56:35.253587Z","shell.execute_reply":"2023-07-20T23:56:35.263603Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by https://infovis.fh-potsdam.de/tutorials/infovis5text.html\n\n# no punctuation, numbers or contractions\nonlywords = [word for word in words if word.isalpha()]\n\nonlywords[0:20]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:56:56.117994Z","iopub.execute_input":"2023-07-20T23:56:56.118565Z","iopub.status.idle":"2023-07-20T23:56:56.126327Z","shell.execute_reply.started":"2023-07-20T23:56:56.118527Z","shell.execute_reply":"2023-07-20T23:56:56.125157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words = [word for word in words if word.isalpha()]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:57:06.790390Z","iopub.execute_input":"2023-07-20T23:57:06.791367Z","iopub.status.idle":"2023-07-20T23:57:06.796428Z","shell.execute_reply.started":"2023-07-20T23:57:06.791326Z","shell.execute_reply":"2023-07-20T23:57:06.795332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by https://infovis.fh-potsdam.de/tutorials/infovis5text.html\n\n# to save us some typing, we import these, so we can call them directly\nfrom nltk import word_tokenize, pos_tag\n\ntokens = word_tokenize(text.lower())\nwords = [word for word in tokens if word.isalpha()]\n\n# bag of words as a dictionary data type\nbow = {}\n\n# we count the occurrences of each word and save it\nfor word in words:\n bow[word] = words.count(word)\n\n# for later use, we create a sorted list of word-frequency tuples\nwords_frequency = sorted(bow.items(), key=lambda x: x[1], reverse=True)\n\nprint(words_frequency[0:100])","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:57:17.581072Z","iopub.execute_input":"2023-07-20T23:57:17.581465Z","iopub.status.idle":"2023-07-20T23:57:17.589924Z","shell.execute_reply.started":"2023-07-20T23:57:17.581436Z","shell.execute_reply":"2023-07-20T23:57:17.588664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by https://infovis.fh-potsdam.de/tutorials/infovis5text.html\n\nimport altair as alt\n\n# first we create a dataframe from the word frequencies\ndf = pd.DataFrame(words_frequency, columns=['word', 'count'])\n\n# we want to focus just on the top 20 words\ndf_top = df[:20]\n\n# draw horizontal barchart \nalt.Chart(df_top).mark_bar().encode(\n x = 'count:Q',\n y = 'word:N'\n \n)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T23:58:12.204782Z","iopub.execute_input":"2023-07-20T23:58:12.205185Z","iopub.status.idle":"2023-07-20T23:58:12.787513Z","shell.execute_reply.started":"2023-07-20T23:58:12.205156Z","shell.execute_reply":"2023-07-20T23:58:12.786456Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Only two lines with Altair","metadata":{}},{"cell_type":"code","source":"from matplotlib.font_manager import FontProperties\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.axes_style(\"white\")\n\ndef barh_plot(dataframe, column):\n prop = FontProperties(fname='kalpurush.ttf', size=20)\n _, ax = plt.subplots(figsize=(12, 8))\n sns.countplot(data=dataframe, x=column,\n order=dataframe[column].value_counts().index, color='teal')\n ax.bar_label(ax.containers[0])\n ax.set_xticklabels(ax.get_xticklabels(), fontproperties=prop)\n ax.tick_params(axis='x', rotation=45)\n ax.set_title('Bengali Sentences')\n plt.savefig('sentence_count', dpi=100, bbox_inches='tight')\n\n\nbarh_plot(sub, 'sentence')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:44:00.158293Z","iopub.execute_input":"2023-07-21T00:44:00.158682Z","iopub.status.idle":"2023-07-21T00:44:00.544283Z","shell.execute_reply.started":"2023-07-21T00:44:00.158657Z","shell.execute_reply":"2023-07-21T00:44:00.543417Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define a function to plot a bar plot easily\n\ndef bar_plot(sub,x,x_title,y,title,colors=None,text=None):\n fig = px.bar(x=x,\n y=y,\n text=text,\n labels={x: x_title.title()}, # replaces default labels by column name\n data_frame=sub,\n color=colors,\n barmode='group',\n template=\"simple_white\",\n color_discrete_sequence=px.colors.qualitative.Prism)\n \n texts = [sub[col].values for col in y]\n for i, t in enumerate(texts):\n fig.data[i].text = t\n fig.data[i].textposition = 'inside'\n \n fig['layout'].title=title\n\n for trace in fig.data:\n trace.name = trace.name.replace('_',' ').title()\n\n fig.update_yaxes(tickprefix=\"\", showgrid=True)\n\n fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:46:05.427676Z","iopub.execute_input":"2023-07-21T00:46:05.428028Z","iopub.status.idle":"2023-07-21T00:46:05.434910Z","shell.execute_reply.started":"2023-07-21T00:46:05.428001Z","shell.execute_reply":"2023-07-21T00:46:05.433914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets define a function to plot a histogram plot easily\n\ndef hist_plot(sub,x,title):\n fig = px.histogram(x=sub[x],\n color_discrete_sequence=colors,\n opacity=0.8)\n\n fig['layout'].title=title\n fig.update_yaxes(tickprefix=\"\", showgrid=True)\n fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:46:41.015674Z","iopub.execute_input":"2023-07-21T00:46:41.016020Z","iopub.status.idle":"2023-07-21T00:46:41.022156Z","shell.execute_reply.started":"2023-07-21T00:46:41.015994Z","shell.execute_reply":"2023-07-21T00:46:41.020911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Alaa Sedeeq https://www.kaggle.com/alaasedeeq/commonlit-readability-eda/comments#1292516\n\nimport plotly.express as px\n\n#Find words spreading (each word frequency)\nfreq_d = pd.Series(' '.join(sub['sentence']).split()).value_counts()\n#Plot the words distribution\nfig = px.line(freq_d,\n title='The word frequency visualization')\nfig.update_layout(showlegend=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:47:23.673711Z","iopub.execute_input":"2023-07-21T00:47:23.674039Z","iopub.status.idle":"2023-07-21T00:47:25.094636Z","shell.execute_reply.started":"2023-07-21T00:47:23.674015Z","shell.execute_reply":"2023-07-21T00:47:25.093419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prepared_as_text = [line for line in sub['sentence']]\ntext_prepared_results = '/n'.join(prepared_as_text)\n\ntext= ' '.join(t for t in sub['sentence'])\nwords_list= text.split()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:48:25.952112Z","iopub.execute_input":"2023-07-21T00:48:25.952481Z","iopub.status.idle":"2023-07-21T00:48:25.957268Z","shell.execute_reply.started":"2023-07-21T00:48:25.952455Z","shell.execute_reply":"2023-07-21T00:48:25.956598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_freq= {}\n\nfor word in set(words_list):\n word_freq[word]= words_list.count(word)\n \n#sorting the dictionary \nword_freq = dict(sorted(word_freq.items(), reverse=True, key=lambda item: item[1]))","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:48:47.813716Z","iopub.execute_input":"2023-07-21T00:48:47.814036Z","iopub.status.idle":"2023-07-21T00:48:47.818691Z","shell.execute_reply.started":"2023-07-21T00:48:47.814012Z","shell.execute_reply":"2023-07-21T00:48:47.817808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sort the data and put it in a data frame for the visualization\nimport itertools\n\nword_freq_temp = dict(itertools.islice(word_freq.items(), 25))\nword_freq_df = pd.DataFrame(word_freq_temp.items(),columns=['word','count']).sort_values('count',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:49:09.431808Z","iopub.execute_input":"2023-07-21T00:49:09.432137Z","iopub.status.idle":"2023-07-21T00:49:09.439635Z","shell.execute_reply.started":"2023-07-21T00:49:09.432112Z","shell.execute_reply":"2023-07-21T00:49:09.438169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Top Frequent Words","metadata":{}},{"cell_type":"code","source":"#Code by Alaa Sedeeq https://www.kaggle.com/alaasedeeq/commonlit-readability-eda/comments#1292516\n\nbar_plot(word_freq_df.reset_index(),\n 'word',\n 'Words',\n ['count'],\n title='Top 20 frequent words')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:49:34.404082Z","iopub.execute_input":"2023-07-21T00:49:34.404430Z","iopub.status.idle":"2023-07-21T00:49:34.568944Z","shell.execute_reply.started":"2023-07-21T00:49:34.404398Z","shell.execute_reply":"2023-07-21T00:49:34.567792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk \nimport string\nfrom wordcloud import WordCloud\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\nfrom nltk.stem import WordNetLemmatizer\nfrom textblob import TextBlob,Word\nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:50:28.079249Z","iopub.execute_input":"2023-07-21T00:50:28.079604Z","iopub.status.idle":"2023-07-21T00:50:28.936486Z","shell.execute_reply.started":"2023-07-21T00:50:28.079577Z","shell.execute_reply":"2023-07-21T00:50:28.935341Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Alaa Sedeeq https://www.kaggle.com/alaasedeeq/commonlit-readability-eda/comments#1292516\n#Bigrams\nfrom nltk.util import ngrams \n\ntext1= 'আমি কোন পার্থক্য দেখতে না,একজন পুরুষ এবং মহিলার মধ্যে,যাই হোক না কেন মহান বা পরোপকারী অর্জন,যে এই পৃথিবীতে আছে, এর অর্ধেক ছিল নারীর দ্বারা, বাকি অর্ধেক মানুষের দ্বারা।'.join(t for t in sub['sentence'])\n\ndef get_n_grans_count(text1, n_grams, min_freq):\n output = {}\n tokens = nltk.word_tokenize(text)\n\n #Create the n_gram\n if n_grams == 2:\n gs = nltk.bigrams(tokens)\n \n elif n_grams == 3:\n gs = nltk.trigrams(tokens)\n\n else:\n return 'Only 2_grams and 3_grams are supported'\n #compute frequency distribution for all the bigrams in the text\n fdist = nltk.FreqDist(gs)\n for k,v in fdist.items():\n if v > min_freq:\n index = ' '.join(k)\n output[index] = v\n \n return output","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:14:35.702019Z","iopub.execute_input":"2023-07-21T01:14:35.702437Z","iopub.status.idle":"2023-07-21T01:14:35.715803Z","shell.execute_reply.started":"2023-07-21T01:14:35.702404Z","shell.execute_reply":"2023-07-21T01:14:35.714729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install langdetect # Language Detection\n!pip install bnlp_toolkit # For Bangla Word Cloud\n!wget https://www.omicronlab.com/download/fonts/kalpurush.ttf # Bangla Font For the Word Cloud","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:22:05.980187Z","iopub.execute_input":"2023-07-21T01:22:05.980548Z","iopub.status.idle":"2023-07-21T01:22:30.968017Z","shell.execute_reply.started":"2023-07-21T01:22:05.980523Z","shell.execute_reply":"2023-07-21T01:22:30.966748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def printmd(string):\n display(Markdown(string))\n\nfrom langdetect import detect\nimport unicodedata\nimport html","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:22:44.839388Z","iopub.execute_input":"2023-07-21T01:22:44.839727Z","iopub.status.idle":"2023-07-21T01:22:44.856899Z","shell.execute_reply.started":"2023-07-21T01:22:44.839702Z","shell.execute_reply":"2023-07-21T01:22:44.855105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wordcloud(text,stopwords,ngram=1):\n # text: if ngram>1, text should be a dictionary\n wordcloud = WordCloud(width=1400, \n height=800,\n random_state=2021,\n background_color='black',\n colormap='Set2',\n font_path=\"./kalpurush.ttf\",\n stopwords=stop)\n if ngram ==1:\n wordc = wordcloud.generate(' '.join(text))\n else:\n wordc = wordcloud.generate_from_frequencies(text)\n plt.figure(figsize=(20,10), facecolor='k')\n plt.imshow(wordcloud)\n plt.axis('off')\n plt.tight_layout(pad=0)\n \nwordcloud(sub['sentence'],stop)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:23:18.311260Z","iopub.execute_input":"2023-07-21T01:23:18.311605Z","iopub.status.idle":"2023-07-21T01:23:19.414627Z","shell.execute_reply.started":"2023-07-21T01:23:18.311581Z","shell.execute_reply":"2023-07-21T01:23:19.413664Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Trigrams","metadata":{}},{"cell_type":"code","source":"three_grams = get_n_grans_count(text, n_grams=3, min_freq=0)\nthree_grams_df = pd.DataFrame(data=three_grams.items())\nthree_grams_df = three_grams_df.sort_values(by=1,ascending=False).rename(columns={0:'Three grams',1:'Count'})\nthree_grams_df","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:24:29.235517Z","iopub.execute_input":"2023-07-21T01:24:29.235886Z","iopub.status.idle":"2023-07-21T01:24:29.249848Z","shell.execute_reply.started":"2023-07-21T01:24:29.235858Z","shell.execute_reply":"2023-07-21T01:24:29.248913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Why Trigram worked and Bigram did Not? It returned Key error = 1 ","metadata":{}},{"cell_type":"code","source":"bar_plot(three_grams_df.iloc[:20],\n 'Three grams',\n 'Three grams',\n ['Count'],\n title='Top 20 frequent trigram')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:25:21.085165Z","iopub.execute_input":"2023-07-21T01:25:21.085568Z","iopub.status.idle":"2023-07-21T01:25:21.151549Z","shell.execute_reply.started":"2023-07-21T01:25:21.085537Z","shell.execute_reply":"2023-07-21T01:25:21.150563Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"three_grams_temp = {j.replace(' ','_') : k for j, k in three_grams.items()}\n\nwordcloud(three_grams_temp,stop,ngram=3)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:27:23.624829Z","iopub.execute_input":"2023-07-21T01:27:23.625192Z","iopub.status.idle":"2023-07-21T01:27:25.135554Z","shell.execute_reply.started":"2023-07-21T01:27:23.625164Z","shell.execute_reply":"2023-07-21T01:27:25.134359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Words length","metadata":{}},{"cell_type":"code","source":"words_length = {}\n\nfor word in set(words_list):\n words_length[word] = len(word)\n \nwords_length = dict(sorted(words_length.items(), reverse=True, key=lambda item: item[1]))\n#sort the data and put it in a data frame for the visualization\nword_length_temp = dict(itertools.islice(words_length.items(), 25))\nwords_length_df = pd.DataFrame(words_length.items(),columns=['word','count']).sort_values('count',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:28:05.432383Z","iopub.execute_input":"2023-07-21T01:28:05.432729Z","iopub.status.idle":"2023-07-21T01:28:05.440913Z","shell.execute_reply.started":"2023-07-21T01:28:05.432702Z","shell.execute_reply":"2023-07-21T01:28:05.439732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Sentence level analysis\n\nSentence level analysis Text statistics include sentence length distribution, minimum, maximum, and average length. To check the sentence length distribution. Code and output are as follows:","metadata":{}},{"cell_type":"code","source":"train['sentence_len']= train['sentence'].str.len()\nprint('Max length : {} \\nMin length : {} \\nAverage Length : {}'.\\\n format(max(train['sentence_len']),min(train['sentence_len']),train['sentence_len'].mean()))","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:29:38.638633Z","iopub.execute_input":"2023-07-21T01:29:38.638959Z","iopub.status.idle":"2023-07-21T01:29:39.304340Z","shell.execute_reply.started":"2023-07-21T01:29:38.638935Z","shell.execute_reply":"2023-07-21T01:29:39.303240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#The longest sentence","metadata":{}},{"cell_type":"code","source":"#the longest sentence we have\ntrain[train['sentence_len']==max(train['sentence_len'])]['sentence'].values[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:30:45.233321Z","iopub.execute_input":"2023-07-21T01:30:45.233667Z","iopub.status.idle":"2023-07-21T01:30:45.328568Z","shell.execute_reply.started":"2023-07-21T01:30:45.233640Z","shell.execute_reply":"2023-07-21T01:30:45.327442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#The shortest sentence","metadata":{}},{"cell_type":"code","source":"#the shortest sentence we have\ntrain[train['sentence_len']==min(train['sentence_len'])]['sentence'].values[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:32:02.105746Z","iopub.execute_input":"2023-07-21T01:32:02.106132Z","iopub.status.idle":"2023-07-21T01:32:02.198959Z","shell.execute_reply.started":"2023-07-21T01:32:02.106094Z","shell.execute_reply":"2023-07-21T01:32:02.197558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = px.colors.qualitative.Prism\n\nhist_plot(train,\n 'sentence_len',\n title='Sentences lenght distribution with spaces')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:32:46.819931Z","iopub.execute_input":"2023-07-21T01:32:46.820229Z","iopub.status.idle":"2023-07-21T01:32:47.186966Z","shell.execute_reply.started":"2023-07-21T01:32:46.820207Z","shell.execute_reply":"2023-07-21T01:32:47.185624Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = px.colors.qualitative.Prism\n\n# lets define a function to plot a histogram plot easily\n\ndef hist_plot(train,x,title):\n fig = px.histogram(x=train[x],\n color_discrete_sequence=colors,\n opacity=0.8)\n\n fig['layout'].title=title\n fig.update_yaxes(tickprefix=\"\", showgrid=True)\n fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:33:59.720074Z","iopub.execute_input":"2023-07-21T01:33:59.720427Z","iopub.status.idle":"2023-07-21T01:33:59.726086Z","shell.execute_reply.started":"2023-07-21T01:33:59.720400Z","shell.execute_reply":"2023-07-21T01:33:59.725418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = px.colors.qualitative.Prism\n\ntrain['sentence_len_no_sp']= train['sentence'].str.split().map(lambda x: len(x))\n\nhist_plot(train,\n 'sentence_len_no_sp',\n title='Sentences lengh distribution without spaces')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:34:59.804768Z","iopub.execute_input":"2023-07-21T01:34:59.805134Z","iopub.status.idle":"2023-07-21T01:35:03.391604Z","shell.execute_reply.started":"2023-07-21T01:34:59.805106Z","shell.execute_reply":"2023-07-21T01:35:03.390343Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#It's a pity that we still don't have Bengali Spacy","metadata":{}},{"cell_type":"markdown","source":"![image.png](attachment:c4c896cc-625a-4066-9ed9-34ded4b5746c.png)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T01:41:13.395363Z","iopub.execute_input":"2023-07-21T01:41:13.395816Z","iopub.status.idle":"2023-07-21T01:41:15.677151Z","shell.execute_reply.started":"2023-07-21T01:41:13.395781Z","shell.execute_reply":"2023-07-21T01:41:15.675334Z"}},"attachments":{"c4c896cc-625a-4066-9ed9-34ded4b5746c.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAApCAYAAAAYunHtAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAzMSURBVHhe7d0NVFRlGgfwP7KAwhof8SH5wSKzAqvQmdwMJVryxOrBVdEVs20rO2rHijidqCXOrqfaTkuadjok1SauZ137EiuBQMgOS5AGGk5ArkB8BMjCIH6QSoJJ+773vjP3DjvqzAAyA8/vnHvm3vt679y5M97nfZ73zuD0EwNCxojm5mYEBweLJWJA54XYG/pM2maCeCSEEEKInaOgTQghhDgICtqEEEKIg6CgTQghhDgICtqEEEKIg6CgTQghhDgICtqEEEKIg6CgTQghhDgICtqEEBMTn39Pmgzz9HjjHgm5HvpFNDKm0K8smUfnhdgb+kzahjJtQogJyvpuPDrnxFKUaZMxxdreO//49/f348qVK2LN8HB2doarqyucnJzEmtFlL+fFXtnb+zUeUKZtG8q0ybg2UoGJ75Pv2xHxrG88BWxutN8vyrSJpShok3FtJAOTowa9S8/fZ/fH3nW8HJlVHWJpeIzma+bnnBBLUNAmhJiw/6zvJD4t1eOSe4BYdnyUaRNLWRe0O7KxIUiDqappQ84p0TjKdOmY+lg2uvi8dJzr8fHwdsTJOKEvegUJu+rE0vhjPuurx7YtOfAR00OHu8X6UfBdC7L6fREXIi5f35bBZ2el/H/fQVGmTSxlQ6a9DrktDWjnU3k6kJxmf8ExMBE7WrKwIlAsE0IsZj7rC0bS47FoZtMHGrFqlFTXnQFm3YJIO6sTPvupDvl17WLJOpRpjyPqBNMGQ/vYB8YifkkJ2jrFMiFjxveo3vMCVt//KFY/8w+U6sVq1OHt+19B9hcHkLoxGQlrn0LqviZcFK1jgfmszwVuHp7wZJOXs1g1WE89MnfnQcuz8a15eORAPXoGRJuUDR/BweJCRG3NQdhrhchq6hWNzIAeB/cVSNuGZRbj4IlKPLSlDEdFs6IZZbVAYmiQWFa0lB80s+9u7NuZg22Hj2PTm3KV4N591WgxHNcQ1Hd/j3OX5JvXzvT24bXDtfi686y0bC3KtMePrtZGIEwDf7FsraEFbV0WkpCONVqxzPFehLF8ri5Rn8LHj2mQodMhw2w7J/8bw/byv1X/G/W2bLpqb2XwdoRY6ZgOJ6Mex663nkPSzDq8uusQlMtxBz7/1hdPvLIZf/9LHPDJduxtEE1jgG1ZXw/yck6gatocFPGM/EENQltPYNOxHtHOXOhGtdetKHo0Gnsjga15OlSLpp5j1bi3wx1b1saiYm04cPIU26MZTf9FNnwQ84tBl64LepROYs+t3rcqMOd9dwmJa9hxrdUgsqMZmV+b3bvF3qlqxtw3DyDrK/mNr9Gfkx4jA7xQ0dYNn5f24hMrsm7KtC1luLarY0U6KkWrZNAwboZOrGe6ctZjKl+hilODh3grM5RtrxZj+H4Gbyft22SIVtmPfAzyMWuTS4BtiWy9bTHKhqC9E8sMB5PQiO2bEpUeAz8RCVDK5/tDkPSi6YvenFCIaNGem1KiaucvaD4KFn0pb8um6LJEbJbaBF0hsF/suyUbqflpeF/1hhAybCIWIF7jC4/JU3DXvfEIralD7SXRBg/8Ju52TJs8CQGae5Bw+w+obRzFMd5hZlvW54mlDy7H2wuD4c8zcv9wPBDhhj3txhIF4OqLVbcFsmzdF5F3Tkdi33nUn5ab6tt7sVQbjjh/ns0HIm62j9xgYkAujYeZKY27B2DVrWLfC9mF+HIPqlUVwJiIOYj05sc1G3+c7YKsNtVxWWHgp5/wTOExrPu4HHEhgXh03ixp/Tddcidgtr8X5rDAffs09lrfK8XzxdWw5JcwKNO2UEcDahCCtuw0YBOPA19i+5KdeMsQQHkMiipCfLmIE+XpqElQgmNbSwniUYgNZYvl9v3rUJCcZQz6PGAvY91COcawfcN8jPGfEYKClpNiiWFB+s/JIch9g8VDHrDVx8CeY/MOHuf8sOINFrcQi+1Sm21DuDYEbdWYdsvjaItSejKVZTsRn7Eec+VFQLuevegiHFL1JlL3pxnb5yamIz6/AW18oaMEBfnrsHG5n9TGzU3mL1BFm4ZkY1avRXQK6+G22smNcGRscXERM4zPFHaZOI2z58UyboKHm5iFCzzcxewYYWvW19dUjuTXlZvVwr7oEy2CqzMmiVm4usBLzBpNUNXd3ZxZN2CQgRaU1fPSuJkf5GBXMuO+2Z6DvC9Drxqz8JpofMMwaaINlz3m+77L+N2/SvB6eR2S54fiw/vugofrz6S2qs6z0vxMn59LjwUP3I0HtTPxculxLHunRNr2WijTtlBnAwpY4ogYQ8Dzw4ww1seeweMGS/x28BiUrgRDkyHcU2itBQpqNXjJEEimaFi7BtP5PAv4y2rToTO0STEpFtOnyIsm+Ha1DcaEtJJ3IqTYx47hxTRE7FcCMo+LxnK46HTMGML9VrZ9eo20SOa9iDIetcUJSZ5vLAlMDZqPpHwLx7z5m2E4eVelLomwHtE2sZoQMmxsyvoG6rH9o1PwjJ4n3azGp/JfK4HSauay0+8MpXGxfIP9Ye8XKG7qRPysqdiy6DaofzztuP6clGUbOE9wwtvL78A9IVNwsKEDD314WLSYR5m2ZaTx4JRsVfKmw6FthsB6Em0syMbPUxI/wzp1u7o63HWkCAUioEr7zk+D1hC/ohqw8WrZcKAGEYaEUwT7l6SEkz8HrygrceqtoC/RbjhgHudSFiuJrQ2GGLTFC5XIPZ5UY/lamZQTfB2Gk2Ag9UoM5PJ52wZlv7ks0yZkRFxWZUZnOtGIm+E9WSyPcTZlfWfPo3rAC8uk8rd8w5obLL/by9/TBXknGlB/kW0zcB5HK/SoEG2yARyt6YZXRBAixRoTbLMfxCxwDi1nXRDgIRaHybur70ScJhAF9e14JKcCVwbkngUvmfMx7TkBSm2Ar0s5UInPGjuxcOYU/PP3C0SLeZRpW4aXt1NjVAFFnbmay2J1hdisbl+yCNGqdqlcHjRNmc9QhmfbW5Sq8P+bhulLGtHKx9ZZdp+6QXQEpGNQV6MbsENVPZaq0eL5bDXEoK3D+8nKSZwbw7LuhEE3BVhKuxipUI1NMLzkUCDmTXtMHO9hiVlChlvFZ8hp6MbF850o/aAAdRGhCJso2sa462V9QX5uLMA2sQDbgx4+8dLvZHfMmnAOucc6pHUtx8uwp9Xyy0vQgnBsgR5RmXnwebUMpd4+uEO0SQYaUPqtC1aFXeWC16PHnir+3N2oLmYXShdPRJoraw7BTW4uyL0/Fml3zcZuXRMW7y7Gxf4f0XjmAvqvDCAywFv6dxfYuhXvfo7Mino8uSAMnzwQK217LZRpW0KdVQvqCi3PflkMOWQcg9YhI0EZslVn1TK5OiyX1uX4pR7fvjaepJagLTsLSWGqzF8qx5vGMYXp89nKhqCtuhEtKBE1rGdiPGBtGnQZjap2Nln8fTQtkqXvfSvl9UMx6jFtLdZkAElRhn0XApRpk5FypxbeRZl4eOML2N4UiqcejoZ8SR77rpf1+c8NxRYnPZZlliCYT4XNgGs4nl7ph9aSI9K6PzX4IOkOK1Jd12CsX7cUHU/EouPJeKTMGhTkGvXY4eGLmFvE8mDevph3ugqL3jyE1dXA00u1I/I9bl4Sf25hJPauicFX7afxn64efCPuHI8IkMvjfMy7uEmPXSvn4+XfajHBgj9CQpm2Bcxk0lLmuihWBGI5htQYS9OJ0o3LhkxXnVVLxH1U0ar4lZuijm/Xvrt7elAsNrPEMdeklOyHFZtM45h0t7poi2bHKpXOh/A9bfv+K1/SXXh8XOFaZQpCFNb+5aDeXtV3hUeAu7t93KVmb+fFrD6Wtf/IZ3pxpOgrPAENqlaGww19KP2oEJk+8/BBrLr2Ofxseb/++u8a/O3zb6B/dhU8J7rg7A/96O7twy9vHifjKTZy7L/yJQ/X8m87qcvfN8II9EWHi3wX3lAH7Qkh1hmtrO9oocjcM49g6wU/7F3CAzbT34SyJhcs/dXIBmxb3XaLD1Kiw6WAzXlPcrU6YFOm7Vi6ctKk3yiRbz67sewo05Z7Lkn5YpFLyVbuuiPEApRpm+cQmbYdsJf3azxwxEyb/4CK9OMooxib7Ls8ToiVrL0Q9PX1jdifZHR2doab2xC+9jSMrDkvPOvrSVvpsH9a1Faj+X7xcz7ebkZzxKBtD+y4PE7IyHN1dZUu1sON75Pv2xHx4DFS58Vejfb7RXePE0tR0CbjmpOTk5Rd8bLocE58n3zfjohnfZNeeF96DfyXzfjrGeuPnukfSe+XYWx5tB4JuR4qj5MxhUpu5tF5IfaGPpO2oUybEEIIcRAUtAkhhBAHQUGbEEIIcRAUtAkhhBAHQUGbEEIIcRAUtAkhhBAHQUGbEEIIcRAUtAkhhBCHAPwPQwjWBzHLORUAAAAASUVORK5CYII="}}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nAlaa Sedeeq https://www.kaggle.com/code/alaasedeeq/commonlit-readability-eda/notebook\n\nmpwolke https://www.kaggle.com/code/mpwolke/rock-pop-lyrics","metadata":{}}]}