{"cells":[{"metadata":{"trusted":true},"cell_type":"markdown","source":"## About\n\nThis Notebook generates LDA feature used in the [9th place LightGBM](https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/210354) with 20M rows of data.\n\nIn the actual solution, I ran this script in GCP with all rows (due to [this behavior](https://github.com/pandas-dev/pandas/issues/26314), pandas version must be 0.21 or earlier to run this script with all rows)."},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.decomposition import LatentDirichletAllocation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_feather('/kaggle/input/riiidtrainfeather/train.f', \n                     columns=['user_id', 'content_id', 'content_type_id', 'answered_correctly']).head(20000000)\nq = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\n\ndf = df[(df['answered_correctly'] == 0) & (df['content_type_id'] == 0)].drop('answered_correctly', axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mat = df.groupby(['content_id', 'user_id'])['content_type_id'].count()\nmat = mat.astype(np.uint8)\nmat = mat.unstack(fill_value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LDA_DIM = 10\n\nlda = LatentDirichletAllocation(LDA_DIM, random_state=0)\ntransformed = lda.fit_transform(mat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transformed_df = pd.DataFrame(transformed)\ntransformed_df.columns = [f'lda_item_inc_{i}' for i in range(LDA_DIM)]\ntransformed_df = transformed_df.astype(np.float32)\ntransformed_df['question_id'] = mat.index.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qmat = pd.merge(q[['question_id']], transformed_df, on='question_id', how='left')\nqmat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qmat.columns = ['question_id'] + [f'lda_item_inc_{i}' for i in range(LDA_DIM)]\nqmat.to_feather('question_lda_incorrect_mat.f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}