{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nimport cv2\nimport skimage\nimport IPython.display\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nfrom glob import glob\n\nfrom collections import OrderedDict\nimport torch\nimport gc\nimport cuml\n\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-29T23:28:58.131784Z","iopub.execute_input":"2022-04-29T23:28:58.132117Z","iopub.status.idle":"2022-04-29T23:29:03.982739Z","shell.execute_reply.started":"2022-04-29T23:28:58.132039Z","shell.execute_reply":"2022-04-29T23:29:03.982027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/hm-emb-clip-vit32","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:29:03.986387Z","iopub.execute_input":"2022-04-29T23:29:03.986586Z","iopub.status.idle":"2022-04-29T23:29:04.667099Z","shell.execute_reply.started":"2022-04-29T23:29:03.986563Z","shell.execute_reply":"2022-04-29T23:29:04.666295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv', dtype='str')\nart['detail_desc'] = art['detail_desc'].apply(lambda x: x.lower() if x==x else 'no description' )\nart['path'] = art['article_id'].apply(lambda x: '../input/h-and-m-personalized-fashion-recommendations/images/' + x[:3] + '/' + x + '.jpg')\nart.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:15.941424Z","iopub.execute_input":"2022-04-29T23:42:15.941685Z","iopub.status.idle":"2022-04-29T23:42:16.657887Z","shell.execute_reply.started":"2022-04-29T23:42:15.941655Z","shell.execute_reply":"2022-04-29T23:42:16.657054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.loc[art['department_no']=='8398' , 'department_no' ] = '8316'\nart.loc[art['department_no']=='1620' , 'department_no' ] = '1522'\nart.loc[art['department_no']=='1727' , 'department_no' ] = '1522'\nart.loc[art['department_no']=='3931' , 'department_no' ] = '3932'\nart.loc[art['department_no']=='3938' , 'department_no' ] = '3936'\nart.loc[art['department_no']=='3939' , 'department_no' ] = '3936'\nart.loc[art['department_no']=='3943' , 'department_no' ] = '5683'\nart.loc[art['department_no']=='5787' , 'department_no' ] = '5831'\nart.loc[art['department_no']=='5454' , 'department_no' ] = '5631'\nart.loc[art['department_no']=='7659' , 'department_no' ] = '7658'\nart.loc[art['department_no']=='2930' , 'department_no' ] = '2932'\nart.loc[art['department_no']=='7854' , 'department_no' ] = '7848'\nart.loc[art['department_no']=='7819' , 'department_no' ] = '7917'\nart.loc[art['department_no']=='3510' , 'department_no' ] = '1710'\nart.loc[art['department_no']=='7989' , 'department_no' ] = '7988'\nart.loc[art['department_no']=='2932' , 'department_no' ] = '7814'\nart.loc[art['department_no']=='8564' , 'department_no' ] = '8563'","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:16.659540Z","iopub.execute_input":"2022-04-29T23:42:16.659874Z","iopub.status.idle":"2022-04-29T23:42:16.938528Z","shell.execute_reply.started":"2022-04-29T23:42:16.659835Z","shell.execute_reply":"2022-04-29T23:42:16.937849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:16.939859Z","iopub.execute_input":"2022-04-29T23:42:16.940134Z","iopub.status.idle":"2022-04-29T23:42:16.945444Z","shell.execute_reply.started":"2022-04-29T23:42:16.940099Z","shell.execute_reply":"2022-04-29T23:42:16.944716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art['index_name'].value_counts().tail(50)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:16.947351Z","iopub.execute_input":"2022-04-29T23:42:16.947957Z","iopub.status.idle":"2022-04-29T23:42:16.970819Z","shell.execute_reply.started":"2022-04-29T23:42:16.947894Z","shell.execute_reply":"2022-04-29T23:42:16.969978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art['index_group_name'].value_counts().tail(50)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:17.832984Z","iopub.execute_input":"2022-04-29T23:42:17.833660Z","iopub.status.idle":"2022-04-29T23:42:17.855034Z","shell.execute_reply.started":"2022-04-29T23:42:17.833612Z","shell.execute_reply":"2022-04-29T23:42:17.853743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.loc[art['section_name']=='Ladies Other' , 'section_name'] = 'Womens Everyday Collection'\nart['section_name'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:18.172294Z","iopub.execute_input":"2022-04-29T23:42:18.172507Z","iopub.status.idle":"2022-04-29T23:42:18.210226Z","shell.execute_reply.started":"2022-04-29T23:42:18.172481Z","shell.execute_reply":"2022-04-29T23:42:18.209603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(art.path.values[0])\nprint(img.shape)\nimg = cv2.resize(img, (256,256))\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:20.330205Z","iopub.execute_input":"2022-04-29T23:42:20.330459Z","iopub.status.idle":"2022-04-29T23:42:20.684199Z","shell.execute_reply.started":"2022-04-29T23:42:20.330431Z","shell.execute_reply":"2022-04-29T23:42:20.683519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMB  = np.load('../input/hm-emb-clip-vit32/text0-emb.npy')\nEMB += np.load('../input/hm-emb-clip-vit32/text1-emb.npy')\nEMB += np.load('../input/hm-emb-clip-vit32/text2-emb.npy')\nEMB += np.load('../input/hm-emb-clip-vit32/text3-emb.npy')\nEMB += np.load('../input/hm-emb-clip-vit32/text4-emb.npy')\nEMB = 20 * np.load('../input/hm-emb-clip-vit32/image-emb.npy')\nEMB /= 25.\nEMB.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:21.396385Z","iopub.execute_input":"2022-04-29T23:42:21.396820Z","iopub.status.idle":"2022-04-29T23:42:21.542302Z","shell.execute_reply.started":"2022-04-29T23:42:21.396780Z","shell.execute_reply":"2022-04-29T23:42:21.541429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:44:24.079804Z","iopub.execute_input":"2022-04-29T23:44:24.080494Z","iopub.status.idle":"2022-04-29T23:44:24.087244Z","shell.execute_reply.started":"2022-04-29T23:44:24.080456Z","shell.execute_reply":"2022-04-29T23:44:24.086204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:46:29.404240Z","iopub.execute_input":"2022-04-29T23:46:29.404956Z","iopub.status.idle":"2022-04-29T23:46:29.796634Z","shell.execute_reply.started":"2022-04-29T23:46:29.404917Z","shell.execute_reply":"2022-04-29T23:46:29.795764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = art.groupby(['department_no','index_code'])['article_id'].agg('count').reset_index()\ndt[dt['article_id']<=15]","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:48:25.228674Z","iopub.execute_input":"2022-04-29T23:48:25.229366Z","iopub.status.idle":"2022-04-29T23:48:25.269536Z","shell.execute_reply.started":"2022-04-29T23:48:25.229329Z","shell.execute_reply":"2022-04-29T23:48:25.268755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = art.groupby(['department_no','index_code'])['article_id'].agg('count').reset_index()\ndt","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:49:07.361365Z","iopub.execute_input":"2022-04-29T23:49:07.361628Z","iopub.status.idle":"2022-04-29T23:49:07.404272Z","shell.execute_reply.started":"2022-04-29T23:49:07.361599Z","shell.execute_reply":"2022-04-29T23:49:07.403599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:42:28.832138Z","iopub.execute_input":"2022-04-29T23:42:28.832598Z","iopub.status.idle":"2022-04-29T23:42:28.839337Z","shell.execute_reply.started":"2022-04-29T23:42:28.832551Z","shell.execute_reply":"2022-04-29T23:42:28.838602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_neighbors = 5\nINDICES = np.zeros( (art.shape[0], n_neighbors), dtype='int32' )\nind_ref = np.arange(art.shape[0])\n\ndt = art.groupby(['department_no','index_code'])['article_id'].agg('count').reset_index()\nfor n in range(dt.shape[0]):\n    var0 = dt['department_no'].values[n]\n    var1 = dt['index_code'].values[n]\n    print(var0, var1)\n    ind = np.where( (art.department_no.values == var0)&(art.index_code.values == var1) )[0]\n    \n    nn = cuml.neighbors.NearestNeighbors(n_neighbors=n_neighbors)\n    nn.fit(EMB[ind])\n    distances, indices = nn.kneighbors(EMB[ind])\n    ind = ind_ref[ind]\n    indices = ind[indices]\n    INDICES[ind] = indices.astype('int32')\ngc.collect()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-29T23:51:25.164621Z","iopub.execute_input":"2022-04-29T23:51:25.165101Z","iopub.status.idle":"2022-04-29T23:52:12.539395Z","shell.execute_reply.started":"2022-04-29T23:51:25.165063Z","shell.execute_reply":"2022-04-29T23:52:12.538718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INDICES[:15]","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:12.541055Z","iopub.execute_input":"2022-04-29T23:52:12.541308Z","iopub.status.idle":"2022-04-29T23:52:12.546646Z","shell.execute_reply.started":"2022-04-29T23:52:12.541274Z","shell.execute_reply":"2022-04-29T23:52:12.546018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[126,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:12.547997Z","iopub.execute_input":"2022-04-29T23:52:12.548494Z","iopub.status.idle":"2022-04-29T23:52:15.790127Z","shell.execute_reply.started":"2022-04-29T23:52:12.548460Z","shell.execute_reply":"2022-04-29T23:52:15.789529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[111,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:15.791945Z","iopub.execute_input":"2022-04-29T23:52:15.792406Z","iopub.status.idle":"2022-04-29T23:52:17.540226Z","shell.execute_reply.started":"2022-04-29T23:52:15.792371Z","shell.execute_reply":"2022-04-29T23:52:17.539537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[1125,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:40.267117Z","iopub.execute_input":"2022-04-29T23:52:40.267824Z","iopub.status.idle":"2022-04-29T23:52:42.427232Z","shell.execute_reply.started":"2022-04-29T23:52:40.267786Z","shell.execute_reply":"2022-04-29T23:52:42.426546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[1135,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:51.142835Z","iopub.execute_input":"2022-04-29T23:52:51.143289Z","iopub.status.idle":"2022-04-29T23:52:53.203231Z","shell.execute_reply.started":"2022-04-29T23:52:51.143251Z","shell.execute_reply":"2022-04-29T23:52:53.202509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[1225,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:52:58.239045Z","iopub.execute_input":"2022-04-29T23:52:58.239306Z","iopub.status.idle":"2022-04-29T23:53:00.530077Z","shell.execute_reply.started":"2022-04-29T23:52:58.239275Z","shell.execute_reply":"2022-04-29T23:53:00.529398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[1525,:]\nfor i in range(4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:53:05.943484Z","iopub.execute_input":"2022-04-29T23:53:05.944366Z","iopub.status.idle":"2022-04-29T23:53:07.965693Z","shell.execute_reply.started":"2022-04-29T23:53:05.944326Z","shell.execute_reply":"2022-04-29T23:53:07.964273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = INDICES[3125,:]\nfor i in range(1, 4):\n    img = cv2.imread(art.path.values[var[i]])\n    plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:53:12.622259Z","iopub.execute_input":"2022-04-29T23:53:12.622508Z","iopub.status.idle":"2022-04-29T23:53:14.784486Z","shell.execute_reply.started":"2022-04-29T23:53:12.622479Z","shell.execute_reply":"2022-04-29T23:53:14.783622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, 5):\n    art['nn'+str(i-1)] = art['article_id'].values[ INDICES[:,i] ]\n    \nart","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:53:30.382980Z","iopub.execute_input":"2022-04-29T23:53:30.383226Z","iopub.status.idle":"2022-04-29T23:53:30.464231Z","shell.execute_reply.started":"2022-04-29T23:53:30.383198Z","shell.execute_reply":"2022-04-29T23:53:30.463555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art.to_parquet('articles_nn.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T23:53:34.350777Z","iopub.execute_input":"2022-04-29T23:53:34.351398Z","iopub.status.idle":"2022-04-29T23:53:34.815862Z","shell.execute_reply.started":"2022-04-29T23:53:34.351359Z","shell.execute_reply":"2022-04-29T23:53:34.815078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}