{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom wordcloud import WordCloud, STOPWORDS\nfrom PIL import Image\nimport random\n#Text Color\nfrom termcolor import colored\n\n#NLP\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n#WordCloud\nfrom wordcloud import WordCloud, STOPWORDS\n\n#Text Processing\nimport re\nimport nltk","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:36:14.196664Z","iopub.execute_input":"2022-08-11T17:36:14.197162Z","iopub.status.idle":"2022-08-11T17:36:18.308085Z","shell.execute_reply.started":"2022-08-11T17:36:14.197059Z","shell.execute_reply":"2022-08-11T17:36:18.306595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/shopee-product-matching/train.csv')\ntest = pd.read_csv('../input/shopee-product-matching/test.csv')\nsample = pd.read_csv('../input/shopee-product-matching/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:42:41.736794Z","iopub.execute_input":"2022-08-11T18:42:41.737339Z","iopub.status.idle":"2022-08-11T18:42:41.855477Z","shell.execute_reply.started":"2022-08-11T18:42:41.737298Z","shell.execute_reply":"2022-08-11T18:42:41.854614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of images in train dataset:\",len(train))\nprint(\"Number of images in test dataset:\",len(test))","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:05.293405Z","iopub.execute_input":"2022-08-11T17:37:05.293829Z","iopub.status.idle":"2022-08-11T17:37:05.300746Z","shell.execute_reply.started":"2022-08-11T17:37:05.293784Z","shell.execute_reply":"2022-08-11T17:37:05.299422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Image Folder Paths\ntrain_jpg_directory = '../input/shopee-product-matching/train_images'\ntest_jpg_directory = '../input/shopee-product-matching/test_images'","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:04.603225Z","iopub.execute_input":"2022-08-11T17:38:04.603701Z","iopub.status.idle":"2022-08-11T17:38:04.608997Z","shell.execute_reply.started":"2022-08-11T17:38:04.603661Z","shell.execute_reply":"2022-08-11T17:38:04.608031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getImagePaths(path):\n    \"\"\"\n    Function to Combine Directory Path with individual Image Paths\n    \n    parameters: path(string) - Path of directory\n    returns: image_names(string) - Full Image Path\n    \"\"\"\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:06.387909Z","iopub.execute_input":"2022-08-11T17:38:06.388388Z","iopub.status.idle":"2022-08-11T17:38:06.394966Z","shell.execute_reply.started":"2022-08-11T17:38:06.388351Z","shell.execute_reply":"2022-08-11T17:38:06.394088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get complete image paths for train and test datasets\ntrain_images_path = getImagePaths(train_jpg_directory)\ntest_images_path = getImagePaths(test_jpg_directory)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:09.005765Z","iopub.execute_input":"2022-08-11T17:38:09.007033Z","iopub.status.idle":"2022-08-11T17:38:31.215366Z","shell.execute_reply.started":"2022-08-11T17:38:09.006986Z","shell.execute_reply":"2022-08-11T17:38:31.214002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images_path","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:35.865964Z","iopub.execute_input":"2022-08-11T17:38:35.867225Z","iopub.status.idle":"2022-08-11T17:38:35.873519Z","shell.execute_reply.started":"2022-08-11T17:38:35.867181Z","shell.execute_reply":"2022-08-11T17:38:35.872637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:28.909360Z","iopub.execute_input":"2022-08-11T17:37:28.910557Z","iopub.status.idle":"2022-08-11T17:37:28.937594Z","shell.execute_reply.started":"2022-08-11T17:37:28.910507Z","shell.execute_reply":"2022-08-11T17:37:28.936369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:31.056047Z","iopub.execute_input":"2022-08-11T17:37:31.056921Z","iopub.status.idle":"2022-08-11T17:37:31.070104Z","shell.execute_reply.started":"2022-08-11T17:37:31.056870Z","shell.execute_reply":"2022-08-11T17:37:31.068617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Checking missing data\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:33.355161Z","iopub.execute_input":"2022-08-11T17:37:33.355964Z","iopub.status.idle":"2022-08-11T17:37:33.371484Z","shell.execute_reply.started":"2022-08-11T17:37:33.355925Z","shell.execute_reply":"2022-08-11T17:37:33.370238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:35.822818Z","iopub.execute_input":"2022-08-11T17:37:35.823856Z","iopub.status.idle":"2022-08-11T17:37:35.831641Z","shell.execute_reply.started":"2022-08-11T17:37:35.823815Z","shell.execute_reply":"2022-08-11T17:37:35.830325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Column-wise unique values\nfor col in train.columns:\n    print(col + \":\" + colored(str(len(train[col].unique()))))","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:37:37.939802Z","iopub.execute_input":"2022-08-11T17:37:37.940307Z","iopub.status.idle":"2022-08-11T17:37:37.988823Z","shell.execute_reply.started":"2022-08-11T17:37:37.940258Z","shell.execute_reply":"2022-08-11T17:37:37.987498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to display images:\ndef display_multiple_img(images_paths, rows, cols):\n    \"\"\"\n    Function to Display Images from Dataset.\n    \n    parameters: images_path(string) - Paths of Images to be displayed\n                rows(int) - No. of Rows in Output\n                cols(int) - No. of Columns in Output\n    \"\"\"\n    figure, ax = plt.subplots(nrows=rows,ncols=cols,figsize=(16,8) )\n    for ind,image_path in enumerate(images_paths):\n        image=cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) \n        try:\n            ax.ravel()[ind].imshow(image)\n            ax.ravel()[ind].set_axis_off()\n        except:\n            continue;\n    plt.tight_layout()\n    plt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:26:10.292457Z","iopub.execute_input":"2022-08-11T18:26:10.292873Z","iopub.status.idle":"2022-08-11T18:26:10.301445Z","shell.execute_reply.started":"2022-08-11T18:26:10.292837Z","shell.execute_reply":"2022-08-11T18:26:10.300080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_multiple_img(train_images_path[10:30], 3,5)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:40.984368Z","iopub.execute_input":"2022-08-11T17:38:40.985478Z","iopub.status.idle":"2022-08-11T17:38:43.344374Z","shell.execute_reply.started":"2022-08-11T17:38:40.985433Z","shell.execute_reply":"2022-08-11T17:38:43.343323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_multiple_img(test_images_path, 1, 3)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:38:48.521174Z","iopub.execute_input":"2022-08-11T17:38:48.522328Z","iopub.status.idle":"2022-08-11T17:38:49.193236Z","shell.execute_reply.started":"2022-08-11T17:38:48.522278Z","shell.execute_reply":"2022-08-11T17:38:49.192057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top10 = pd.DataFrame(train.label_group.value_counts().head(10))\ntop10.reset_index(inplace=True)\ntop10.columns = ['label_group','count']\ntop10","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:39:59.632716Z","iopub.execute_input":"2022-08-11T17:39:59.633155Z","iopub.status.idle":"2022-08-11T17:39:59.647365Z","shell.execute_reply.started":"2022-08-11T17:39:59.633119Z","shell.execute_reply":"2022-08-11T17:39:59.646521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the most frequent landmark_ids\ntop20 = pd.DataFrame(train.label_group.value_counts().head(20))\ntop20.reset_index(inplace=True)\ntop20.columns = ['label_group','count']\ntop20\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.figure(figsize = (22, 8))\nplt.title('Most Frequent Landmarks')\nsns.set_color_codes(\"muted\")\nsns.barplot(x=\"label_group\", y=\"count\", data=top20, label=\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:51:55.239509Z","iopub.execute_input":"2022-08-11T17:51:55.240836Z","iopub.status.idle":"2022-08-11T17:51:55.608584Z","shell.execute_reply.started":"2022-08-11T17:51:55.240797Z","shell.execute_reply":"2022-08-11T17:51:55.607455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bot10 = pd.DataFrame(train.label_group.value_counts().tail(10))\nbot10.reset_index(inplace=True)\nbot10.columns = ['label_group','count']\nbot10","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:51:51.024893Z","iopub.execute_input":"2022-08-11T17:51:51.025326Z","iopub.status.idle":"2022-08-11T17:51:51.040676Z","shell.execute_reply.started":"2022-08-11T17:51:51.025283Z","shell.execute_reply":"2022-08-11T17:51:51.039851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the least frequent products\nbot20 = pd.DataFrame(train.label_group.value_counts().tail(20))\nbot20.reset_index(inplace=True)\nbot20.columns = ['label_group','count']\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.figure(figsize = (22, 8))\nplt.title('Most Frequent Landmarks')\nsns.set_color_codes(\"muted\")\nsns.barplot(x=\"label_group\", y=\"count\", data=bot20, label=\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:54:05.165708Z","iopub.execute_input":"2022-08-11T17:54:05.166157Z","iopub.status.idle":"2022-08-11T17:54:05.535054Z","shell.execute_reply.started":"2022-08-11T17:54:05.166119Z","shell.execute_reply":"2022-08-11T17:54:05.533794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Number of products with more than 10 images:\ncounts = train['label_group'].value_counts().sort_values(ascending=False)\nabove10 = counts[counts >10].index.shape[0]\nabove10","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:52:31.134479Z","iopub.execute_input":"2022-08-11T17:52:31.135264Z","iopub.status.idle":"2022-08-11T17:52:31.148233Z","shell.execute_reply.started":"2022-08-11T17:52:31.135194Z","shell.execute_reply":"2022-08-11T17:52:31.146800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Number of lproducts with more than 20 images:\ncounts = train['label_group'].value_counts().sort_values(ascending=False)\nabove10 = counts[counts >20].index.shape[0]\nabove10","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:52:41.124391Z","iopub.execute_input":"2022-08-11T17:52:41.125632Z","iopub.status.idle":"2022-08-11T17:52:41.136816Z","shell.execute_reply.started":"2022-08-11T17:52:41.125587Z","shell.execute_reply":"2022-08-11T17:52:41.135361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set()\nplt.title('Training set: number of images per class(line plot)')\nlandmarks_fold = pd.DataFrame(train['label_group'].value_counts())\nlandmarks_fold.reset_index(inplace=True)\nlandmarks_fold.columns = ['landmark_id','count']\nax = landmarks_fold['count'].plot(logy=True, grid=True)\nlocs, labels = plt.xticks()\nplt.setp(labels, rotation=30)\nax.set(xlabel=\"Products\", ylabel=\"Number of images\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:59:50.736591Z","iopub.execute_input":"2022-08-11T17:59:50.737236Z","iopub.status.idle":"2022-08-11T17:59:51.062485Z","shell.execute_reply.started":"2022-08-11T17:59:50.737201Z","shell.execute_reply":"2022-08-11T17:59:51.061343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Adding file paths to train and test datasets:\ndef get_train_file_path(image_id):\n    return \"../input/shopee-product-matching/train_images/{}\".format(image_id)\ntrain['file_path'] = train['image'].apply(get_train_file_path)\n\n\ndef get_test_file_path(image_id):\n    return \"../input/shopee-product-matching/test_images/{}\".format(image_id)\ntest['file_path'] = test['image'].apply(get_train_file_path)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:43:00.253085Z","iopub.execute_input":"2022-08-11T18:43:00.253583Z","iopub.status.idle":"2022-08-11T18:43:00.281290Z","shell.execute_reply.started":"2022-08-11T18:43:00.253545Z","shell.execute_reply":"2022-08-11T18:43:00.280428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets examine product with label_group is 994676122 which is having highest count :\nimport os\nimport glob\nimport cv2\ntrain1 = train[train.label_group==994676122]\nfig = plt.figure(figsize=(15,15))\nx=1\nfor i in train1.file_path[:16]:\n    image = cv2.imread(i)\n    fig.add_subplot(4, 4, x)\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    x+=1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:56:54.563304Z","iopub.execute_input":"2022-08-11T17:56:54.564049Z","iopub.status.idle":"2022-08-11T17:56:57.141861Z","shell.execute_reply.started":"2022-08-11T17:56:54.564013Z","shell.execute_reply":"2022-08-11T17:56:57.140335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = train[train.label_group==562358068]\nfig = plt.figure(figsize=(15,15))\nx=1\nfor i in train2.file_path[:16]:\n    image = cv2.imread(i)\n    fig.add_subplot(4, 4, x)\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    x+=1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:57:07.827426Z","iopub.execute_input":"2022-08-11T17:57:07.827844Z","iopub.status.idle":"2022-08-11T17:57:09.823296Z","shell.execute_reply.started":"2022-08-11T17:57:07.827810Z","shell.execute_reply":"2022-08-11T17:57:09.822141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train3 = train[train.label_group==3113678103]\nfig = plt.figure(figsize=(15,15))\nx=1\nfor i in train3.file_path[:16]:\n    image = cv2.imread(i)\n    fig.add_subplot(4, 4, x)\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    x+=1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:57:17.645956Z","iopub.execute_input":"2022-08-11T17:57:17.646693Z","iopub.status.idle":"2022-08-11T17:57:20.231638Z","shell.execute_reply.started":"2022-08-11T17:57:17.646652Z","shell.execute_reply":"2022-08-11T17:57:20.230504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain4 = train[train.label_group==1395102007]\nfig = plt.figure(figsize=(15,15))\nx=1\nfor i in train4.file_path[:16]:\n    image = cv2.imread(i)\n    fig.add_subplot(4, 4, x)\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    x+=1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:57:57.163025Z","iopub.execute_input":"2022-08-11T17:57:57.164096Z","iopub.status.idle":"2022-08-11T17:57:57.499530Z","shell.execute_reply.started":"2022-08-11T17:57:57.164044Z","shell.execute_reply":"2022-08-11T17:57:57.498307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain4 = train[train.label_group==3907914144]\nfig = plt.figure(figsize=(15,15))\nx=1\nfor i in train4.file_path[:16]:\n    image = cv2.imread(i)\n    fig.add_subplot(4, 4, x)\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    x+=1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:58:27.642913Z","iopub.execute_input":"2022-08-11T17:58:27.643386Z","iopub.status.idle":"2022-08-11T17:58:27.973463Z","shell.execute_reply.started":"2022-08-11T17:58:27.643346Z","shell.execute_reply":"2022-08-11T17:58:27.972532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Exploring titles:\nprint(train.shape)\nprint(train['title'].nunique())","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:43:50.306220Z","iopub.execute_input":"2022-08-11T18:43:50.306676Z","iopub.status.idle":"2022-08-11T18:43:50.326729Z","shell.execute_reply.started":"2022-08-11T18:43:50.306641Z","shell.execute_reply":"2022-08-11T18:43:50.325330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = train['title'].value_counts().sort_values(ascending=False).reset_index()\nt.columns = ['title','count']\nt","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:44:16.693585Z","iopub.execute_input":"2022-08-11T18:44:16.694728Z","iopub.status.idle":"2022-08-11T18:44:16.733023Z","shell.execute_reply.started":"2022-08-11T18:44:16.694684Z","shell.execute_reply":"2022-08-11T18:44:16.731817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t. loc[t['count'] >1]","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:47:42.822910Z","iopub.execute_input":"2022-08-11T18:47:42.823654Z","iopub.status.idle":"2022-08-11T18:47:42.838317Z","shell.execute_reply.started":"2022-08-11T18:47:42.823616Z","shell.execute_reply":"2022-08-11T18:47:42.837516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Display images for First title = \"Koko syubbanul muslimin koko azzahir koko baju\"\npath=train.loc[train[\"title\"] == \"Koko syubbanul muslimin koko azzahir koko baju\",\"file_path\"]\ndisplay_multiple_img(path,3,3)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:01:30.344622Z","iopub.execute_input":"2022-08-11T19:01:30.345036Z","iopub.status.idle":"2022-08-11T19:01:32.104496Z","shell.execute_reply.started":"2022-08-11T19:01:30.345003Z","shell.execute_reply":"2022-08-11T19:01:32.102937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Monde Boromon Cookies 1 tahun+ 120gr\npath=train.loc[train[\"title\"] == \"Monde Boromon Cookies 1 tahun+ 120gr\",\"file_path\"]\ndisplay_multiple_img(path,2,3)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:03:45.970934Z","iopub.execute_input":"2022-08-11T19:03:45.971661Z","iopub.status.idle":"2022-08-11T19:03:47.135534Z","shell.execute_reply.started":"2022-08-11T19:03:45.971623Z","shell.execute_reply":"2022-08-11T19:03:47.134477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Viva Air Mawar\npath=train.loc[train[\"title\"] == \"Viva Air Mawar\",\"file_path\"]\ndisplay_multiple_img(path,2,3)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:02:51.314728Z","iopub.execute_input":"2022-08-11T19:02:51.315416Z","iopub.status.idle":"2022-08-11T19:02:52.945789Z","shell.execute_reply.started":"2022-08-11T19:02:51.315369Z","shell.execute_reply":"2022-08-11T19:02:52.944346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#wordcloud\nwc = WordCloud(\n    background_color='white',\n    max_words = 150,\n    random_state = 42,\n    max_font_size=80\n    )\nwc.generate(' '.join(train[\"title\"]))\nplt.figure(figsize=(50,7))\nplt.imshow(wc)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T17:59:08.916024Z","iopub.execute_input":"2022-08-11T17:59:08.917015Z","iopub.status.idle":"2022-08-11T17:59:12.101691Z","shell.execute_reply.started":"2022-08-11T17:59:08.916970Z","shell.execute_reply":"2022-08-11T17:59:12.100475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T18:26:30.471560Z","iopub.execute_input":"2022-08-11T18:26:30.472393Z","iopub.status.idle":"2022-08-11T18:26:30.487142Z","shell.execute_reply.started":"2022-08-11T18:26:30.472340Z","shell.execute_reply":"2022-08-11T18:26:30.485708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#First we will work on labels i.e title\n#From the fouth row we can see that the tile is not in english, on investigating it has been found that is in \n#Indonesian langauge\n\nstopwords = nltk.corpus.stopwords.words('english')\nstopwords1 = nltk.corpus.stopwords.words('indonesian')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:10:05.063970Z","iopub.execute_input":"2022-08-11T19:10:05.064536Z","iopub.status.idle":"2022-08-11T19:10:05.072422Z","shell.execute_reply.started":"2022-08-11T19:10:05.064491Z","shell.execute_reply":"2022-08-11T19:10:05.071120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(text):\n    text1=\" \".join([word for word in str(text).split() if word not in stopwords])\n    return \" \".join([word for word in str(text1).split() if word not in stopwords])\ntrain[\"title\"] = train[\"title\"].apply(lambda text: remove_stopwords(text))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:10:06.272997Z","iopub.execute_input":"2022-08-11T19:10:06.274009Z","iopub.status.idle":"2022-08-11T19:10:08.349852Z","shell.execute_reply.started":"2022-08-11T19:10:06.273964Z","shell.execute_reply":"2022-08-11T19:10:08.348578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing Punctuations\nimport string\nenglish_punctuations = string.punctuation\npunctuations_list = english_punctuations\ndef remove_punctuations(text):\n    translator = str.maketrans('', '', punctuations_list)\n    return text.translate(translator)\ntrain[\"title\"] = train[\"title\"].apply(lambda text: remove_punctuations(text))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:10:12.728845Z","iopub.execute_input":"2022-08-11T19:10:12.729737Z","iopub.status.idle":"2022-08-11T19:10:12.906783Z","shell.execute_reply.started":"2022-08-11T19:10:12.729684Z","shell.execute_reply":"2022-08-11T19:10:12.905349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert to lower case\ntrain[\"title\"] = train[\"title\"].str.lower()\n# strip leading and trailing spaces\ntrain[\"title\"] = train[\"title\"].str.strip(' ')\ntrain.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:01:21.768003Z","iopub.execute_input":"2022-08-11T20:01:21.768759Z","iopub.status.idle":"2022-08-11T20:01:21.810210Z","shell.execute_reply.started":"2022-08-11T20:01:21.768709Z","shell.execute_reply":"2022-08-11T20:01:21.809054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gensim\nfrom gensim import corpora, models\n\nfrom pprint import pprint","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:28:23.101654Z","iopub.execute_input":"2022-08-11T19:28:23.102109Z","iopub.status.idle":"2022-08-11T19:28:23.429502Z","shell.execute_reply.started":"2022-08-11T19:28:23.102076Z","shell.execute_reply":"2022-08-11T19:28:23.428094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dictionary and corpus \ndocuments = train[\"title\"].tolist()\n\n# remove common words and tokenize\ntexts = [\n    [word for word in document.lower().split() if word not in stopwords]\n    for document in documents\n]\n\n\n\ndictionary = corpora.Dictionary(texts)\ncorpus = [dictionary.doc2bow(text) for text in texts]","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:28:50.507560Z","iopub.execute_input":"2022-08-11T19:28:50.508044Z","iopub.status.idle":"2022-08-11T19:28:52.893094Z","shell.execute_reply.started":"2022-08-11T19:28:50.508006Z","shell.execute_reply":"2022-08-11T19:28:52.891778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf = models.TfidfModel(corpus)\n\ncorpus_tfidf = tfidf[corpus]\nfor doc in corpus_tfidf:\n    print(doc)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T19:58:25.251030Z","iopub.execute_input":"2022-08-11T19:58:25.251537Z","iopub.status.idle":"2022-08-11T19:58:27.848878Z","shell.execute_reply.started":"2022-08-11T19:58:25.251500Z","shell.execute_reply":"2022-08-11T19:58:27.847705Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TF-IDF VEctorization\nfrom sklearn.feature_extraction.text import TfidfVectorizer  \ntfidf_vect = TfidfVectorizer(min_df=5, max_df=0.7)\n\nX_tfidf = tfidf_vect.fit_transform(train[\"title\"])\nX_f = pd.DataFrame(X_tfidf.toarray())\nX_f.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:19:49.643453Z","iopub.execute_input":"2022-08-11T20:19:49.646320Z","iopub.status.idle":"2022-08-11T20:19:51.168286Z","shell.execute_reply.started":"2022-08-11T20:19:49.646268Z","shell.execute_reply":"2022-08-11T20:19:51.167041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_f=tfidf_vect.transform(test['title'])\ntest_f = pd.DataFrame(test_f.toarray())","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:19:52.545112Z","iopub.execute_input":"2022-08-11T20:19:52.545587Z","iopub.status.idle":"2022-08-11T20:19:52.553451Z","shell.execute_reply.started":"2022-08-11T20:19:52.545548Z","shell.execute_reply":"2022-08-11T20:19:52.552491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn import metrics\nnaive_bayes_classifier = MultinomialNB()\nnaive_bayes_classifier.fit(X_f,train[\"posting_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:20:13.865824Z","iopub.execute_input":"2022-08-11T20:20:13.867196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:20:02.023439Z","iopub.execute_input":"2022-08-11T20:20:02.023865Z","iopub.status.idle":"2022-08-11T20:20:02.029619Z","shell.execute_reply.started":"2022-08-11T20:20:02.023833Z","shell.execute_reply":"2022-08-11T20:20:02.028336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict_test = naive_bayes_classifier.predict(test_f)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:20:03.429559Z","iopub.execute_input":"2022-08-11T20:20:03.430922Z","iopub.status.idle":"2022-08-11T20:20:03.516264Z","shell.execute_reply.started":"2022-08-11T20:20:03.430880Z","shell.execute_reply":"2022-08-11T20:20:03.514640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict_test","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:18:58.116368Z","iopub.execute_input":"2022-08-11T20:18:58.116808Z","iopub.status.idle":"2022-08-11T20:18:58.123191Z","shell.execute_reply.started":"2022-08-11T20:18:58.116759Z","shell.execute_reply":"2022-08-11T20:18:58.122292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:19:10.976505Z","iopub.execute_input":"2022-08-11T20:19:10.977100Z","iopub.status.idle":"2022-08-11T20:19:11.003797Z","shell.execute_reply.started":"2022-08-11T20:19:10.977067Z","shell.execute_reply":"2022-08-11T20:19:11.002908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=naive_bayes_classifier.predict(test_f)\nscore=naive_bayes_classifier.score(test_f,test['posting_id'])\nprint('Accuracy of Naive Bayes :')\nprint(score*100.0)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T20:17:48.511109Z","iopub.execute_input":"2022-08-11T20:17:48.511778Z","iopub.status.idle":"2022-08-11T20:17:49.197856Z","shell.execute_reply.started":"2022-08-11T20:17:48.511741Z","shell.execute_reply":"2022-08-11T20:17:49.196204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = naive_bayes_classifier.predict()\n","metadata":{},"execution_count":null,"outputs":[]}]}