{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook aims to get cluster of questions."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', None)\n\nfrom sklearn.cluster import KMeans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nquestions.head()\nquestions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = questions[~questions['tags'].isna()]\nquestions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = questions.assign(tags=questions['tags'].str.split()).explode('tags')\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"check tag num"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"total tag num is {len(questions['tags'].value_counts())}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = questions[['question_id', 'tags']]\ndf = pd.get_dummies(questions, columns=['tags'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_df = df.groupby('question_id').agg(['sum'])\ntmp_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"km = KMeans(n_clusters=20,\n            init='random',\n            n_init=10,\n            random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = tmp_df.iloc[:, 1:]\ncluster_labels = km.fit_predict(X)\ncluster_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_df['k'] = cluster_labels\ntmp_df = tmp_df.reset_index()\ntmp_df = tmp_df[['question_id', 'k']]\ntmp_df.columns = tmp_df.columns.droplevel(1)\ntmp_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get best cluster num using silhouette score."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import silhouette_score\n\nscores = []\nfor i in range(5, 100, 10):\n    km = KMeans(n_clusters=i,\n                init='random',\n                n_init=10,\n                random_state=0)\n    cluster_labels = km.fit_predict(X)\n    score = silhouette_score(X, cluster_labels)\n    scores.append(score)\n    print(f'cluster num: {i}, silhouette score: {score}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\n\nfig = plt.figure(figsize=(12,6))\npd.Series(scores, index=list(range(5, 100, 10))).plot.bar(scores);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}