{"cells":[{"metadata":{},"cell_type":"markdown","source":"Inspired By This Discussion.\nhttps://www.kaggle.com/c/riiid-test-answer-prediction/discussion/207148\n\nIn the original discussion the idea was to use conditional probability. My idea is to use association analysis. Then using it with KMeans clustering.\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nimport riiideducation\nimport random\nfrom sklearn.metrics import roc_auc_score\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    # 'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    # 'content_type_id':'int8', \n    # 'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    # 'prior_question_elapsed_time': 'float32', \n    # 'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = dt.fread('../input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.astype(data_types_dict)\ntrain_df = train_df[train_df[target] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = train_df[1000000:2000000]\nsample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"limit = 1000\ncontent_agg = train_df.groupby(\"content_id\")[target].agg([\"count\"])\nitems = content_agg[\"count\"].sort_values()[-limit * 2:]\nitems = pd.DataFrame(dict(index_value=items.index, count=items), index=items.index)\nitems = items.sort_values([\"count\", \"index_value\"])[-limit:][[\"count\"]]\nitems","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = sample[sample.content_id.isin(items.index)]\nuser_table = sample.groupby([\"user_id\", \"content_id\"])[target].max()\nuser_table = user_table.unstack()\nuser_table = user_table.fillna(0).astype(\"int8\")\nuser_table","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mlxtend.frequent_patterns import apriori, fpmax, association_rules\n\n\nfreq_items1 = apriori(user_table, min_support=0.01, use_colnames=True, max_len=2)\nfreq_items1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a_rules1 = association_rules(freq_items1, metric=\"confidence\", min_threshold=0)\na_rules1 = a_rules1.sort_values('confidence', ascending=False).reset_index(drop=True)\na_rules1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_ids = items.index[::-1]\nconfidence_df = pd.DataFrame(np.zeros((len(item_ids), len(item_ids))), index=item_ids,\n                            columns=item_ids)\nconfidence_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"values = a_rules1.values\nfor i in range(len(values)):\n    ante = list(values[i, 0])[0]\n    conse = list(values[i, 1])[0]\n    confidence = values[i, 5]\n    # print(conse, ante, confidence)\n    confidence_df.loc[conse, ante] = confidence\nconfidence_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.cluster import KMeans\n\n\nkmeans = KMeans(30, random_state=2)\nkmeans.fit(confidence_df)\n\nlabels = pd.DataFrame(dict(content_id=confidence_df.index, label=kmeans.labels_), index=confidence_df.index)\nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.to_csv(\"labels.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = labels.groupby(\"label\")[\"label\"].count()\nplt.barh(count.index, width=count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confidence_df.loc[labels.label==20, labels[labels.label==20].index]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}