{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_parquet('/kaggle/input/otto-full-optimized-memory-footprint/train.parquet')\ntrain_df = train_df.loc[train_df['type'] == 0]\ntrain_df['active'] = 1\n\nunique_aid = train_df['aid'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom scipy.sparse import csc_matrix\n\nn_session = train_df['session'].nunique()\nn_aid = train_df['aid'].nunique()\n\nsparse_mat = csc_matrix(\n    (train_df['active'].to_numpy(), (train_df['aid'].to_numpy(), train_df['session'].to_numpy())) , \n    shape=(n_aid, n_session),\n    dtype='float32'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy as cp\nfrom cupyx.scipy.sparse import csr_matrix\n\nsparse_mat = csr_matrix( sparse_mat )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom scipy.sparse import csc_matrix\n\nn_session = train_df['session'].nunique()\nn_aid = train_df['aid'].nunique()\n\nsparse_mat_session = csc_matrix(\n    (train_df['active'].to_numpy(), (train_df['session'].to_numpy(), train_df['aid'].to_numpy())) , \n    shape=(n_session, n_aid),\n    dtype='float32'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy as cp\nfrom cupyx.scipy.sparse import csr_matrix\n\nsparse_mat_session = csr_matrix( sparse_mat_session )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def top_20(aid_array):\n    \n    unique_elements, frequency = cp.unique(aid_array, return_counts=True)\n    sorted_indexes = cp.argsort(frequency)[::-1]\n    sorted_by_freq = unique_elements[sorted_indexes]\n\n\n    return sorted_by_freq[:20]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom tqdm import tqdm\n\ni = 0\n\nresult = cp.empty((0,20), dtype='int32')\nfor aid in tqdm(unique_aid, miniters=1):   \n    \n    sessions = sparse_mat[aid,:].indices\n    \n    aid_array = sparse_mat_session[sessions,:].indices\n    top_aid = top_20(aid_array).reshape((1,-1))\n    \n    result = cp.vstack([\n        result,\n        cp.pad(top_aid,((0,0),(0,20-top_aid.shape[1])), constant_values=-1)\n    ])\n\nresult = result.get()\ndel sparse_mat_session, sparse_mat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_name = []\nfor i in range(0,20):\n    column_name.append('top_{}'.format(str(i+1)))\n\ndf = pd.DataFrame(columns = column_name, data = result)\n\ndel result\n\nunique_aid = pd.DataFrame({'aid': unique_aid})\ndf = pd.concat([unique_aid, df], axis=1)\n\ndel unique_aid\n\ndf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_parquet('aid_candidates.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}