{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"general_folder = '/kaggle/input/test-recsys'\nmost_of_tab_subfolder = 'kaggle_tab_1345'\n\n# users_df = pd.read_csv(Path(general_folder, most_of_tab_subfolder, 'tab_4_user_profiles.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.gender.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.gender.fillna('unknown', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df = pd.concat([\n    users_df,\n    pd.get_dummies(users_df.gender, prefix='gender_').set_index(users_df.index)\n], axis=1)\nusers_df.drop('gender', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Preprocess bdate"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Timestamp.min","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_datetime(x):\n    if pd.isnull(x): return x\n    s = x.split('.')\n    if len(s) == 3:\n        return f'{s[2]}-{s[1]}-{s[0]}'\n    elif len(s) == 2:\n        return np.nan\n    else:\n        return x\n\nusers_df['bdate'] = users_df.bdate.apply(fix_datetime)\nusers_df['bdate'] = users_df['bdate'].fillna('1677-09-22')\nusers_df['no_bdate'] = users_df.bdate.apply(lambda x: x == '1677-09-22').astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df['year'] = users_df['bdate'].apply(lambda x: x.split('-')[0])\nusers_df['month'] = users_df['bdate'].apply(lambda x: x.split('-')[1])\nusers_df['day'] = users_df['bdate'].apply(lambda x: x.split('-')[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Give cyclic nature to month, day\n\nusers_df['sin_month'] = users_df.month.apply(lambda x: np.sin(2*np.pi*float(x)/12))\nusers_df['cos_month'] = users_df.month.apply(lambda x: np.cos(2*np.pi*float(x)/12))\nusers_df['sin_day'] = users_df.month.apply(lambda x: np.sin(2*np.pi*float(x)/30))\nusers_df['cos_day'] = users_df.month.apply(lambda x: np.cos(2*np.pi*float(x)/30))\n\nusers_df.drop(['month', 'day'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"b_date_ser = pd.to_datetime(users_df.bdate, format='%Y-%m-%d')\nusers_df.drop('bdate', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"users_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Assembling products and composing dataset for training model sorting prepared mass"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(Path(general_folder, 'sample_submission.csv'))\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime, timedelta\n\ndt = datetime.strptime('2020-08-08', '%Y-%m-%d')\nsber_tab = 'sbermarket_tab_2_6'\nproducts_df = pd.read_csv(Path(general_folder, sber_tab, 'tab_2_products_'+dt.strftime('%Y-%m-%d')+'.csv'))\n\nsber_tab_prefix = 'tab_2_products_'\nfor i in range(3):\n    dt += timedelta(10)\n    products_df = pd.concat([\n        products_df,\n        pd.read_csv(\n            Path(general_folder, \n                 sber_tab, \n                 sber_tab_prefix + dt.strftime('%Y-%m-%d') + '.csv'\n            )\n        )\n    ], axis=0, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.insert(0, 'product_id', products_df.pop('product_id'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df.drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.drop('line_item_id', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# products_df.price = products_df.price / products_df.quantity","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df.dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.drop('quantity', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df = pd.read_csv(Path(general_folder, most_of_tab_subfolder, 'tab_1_orders.csv'))\norders_df = orders_df.drop_duplicates()\norders_df['order_created_time'] = pd.to_datetime(orders_df['order_created_time'], format='%Y-%m-%d %H:%M:%S')\norders_df = orders_df.sort_values('order_created_time', ascending=False)\norders_df = orders_df.drop_duplicates('order_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df.merge(orders_df[['order_id', 'order_created_time']], on='order_id')\nproducts_df = products_df.dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.order_created_time.min(), products_df.order_created_time.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.order_created_time = pd.to_datetime(products_df.order_created_time, format='%Y-%m-%d %H:%M:%S')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lets constraint time interval\n\ndt_from = datetime.strptime('2020-08-14', '%Y-%m-%d')\ndt_to = datetime.strptime('2020-09-14', '%Y-%m-%d')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.order_created_time.min(), orders_df.order_created_time.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df[(products_df.order_created_time >= dt_from) & (products_df.order_created_time <= dt_to)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df = orders_df[\n    (orders_df.order_created_time >= dt_from - timedelta(3)) \\\n    & (orders_df.order_created_time <= dt_to + timedelta(1))\n]\n\nshort_orders_df = orders_df[\n    (orders_df.order_created_time >= dt_from) \\\n    & (orders_df.order_created_time <= dt_to)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.user_id.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nfrom tqdm import tqdm\n\nuser2orders = defaultdict()\n\nfor u in tqdm(products_df.user_id.unique()):\n    ords = orders_df.order_id[orders_df.user_id == u].to_numpy()\n    user2orders[u] = ords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_future(row):\n    usr_id = row.iloc[0]\n    ord_id = row.iloc[1]\n    ords = user2orders[usr_id]\n    cur_ind = np.nonzero(ords == ord_id)[0][0]\n    \n    if cur_ind == 0: return np.nan\n    else:\n        return ords[cur_ind - 1]\n\nproducts_df['fut_order_id'] = (products_df[['user_id', 'order_id']]).apply(lambda row: get_future(row), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_past(row, shift=0):\n    usr_id = row.iloc[0]\n    ord_id = row.iloc[1]\n    ords = user2orders[usr_id]\n    cur_ind = np.nonzero(ords == ord_id)[0][0]\n    \n    if cur_ind + shift >= len(ords) - 1: return np.nan\n    else:\n        return ords[cur_ind + shift + 1]\n    \nproducts_df['p_order_id'] = (products_df[['user_id', 'order_id']]).apply(lambda row: get_past(row, shift=0), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df.dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"order2prods = defaultdict(set)\n\nfor i, row in tqdm(products_df[['product_id', 'order_id']].iterrows()):\n    p = row.iloc[0]\n    o = row.iloc[1]\n    order2prods[o].add(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binarize(row):\n    prod_to_check = row.iloc[0]\n    fut_ord = row.iloc[1]\n    return prod_to_check in order2prods[fut_ord]\n\nproducts_df['target'] = products_df[['product_id', 'fut_order_id']].apply(binarize, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df['prev_purchase'] = products_df[['product_id', 'p_order_id']].apply(binarize, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df['target'] = products_df['target'].apply(lambda x: int(x))\nproducts_df['prev_purchase'] = products_df['prev_purchase'].apply(lambda x: int(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# products_df.drop(['order_id', 'order_created_time', 'fut_order_id', 'p_order_id'], axis=1, inplace=True)\nproducts_df.drop(['order_created_time', 'fut_order_id', 'p_order_id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.prev_purchase.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df = products_df.dropna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# products_df.to_csv(Path('featured_products_unscaled.csv', index=False))\nproducts_df = pd.read_csv(Path('featured_products_unscaled.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.drop('Unnamed: 0', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's use target encoding on features with numerous cats\n\ndef calc_smooth_mean(df, by, on, m):\n    # Compute the global mean\n    mean = df[on].mean()\n\n    # Compute the number of values and the mean of each group\n    agg = df.groupby(by)[on].agg(['count', 'mean'])\n    counts = agg['count']\n    means = agg['mean']\n\n    # Compute the \"smoothed\" means\n    smooth = (counts * means + m * mean) / (counts + m)\n\n    # Replace each value by the according smoothed mean\n    return df[by].map(smooth), smooth\n\nsmooth_mappings = []\nfor col in ['parent_category_id', 'master_category_id']:\n    products_df[col], smooth = calc_smooth_mean(\n        products_df, \n        by=col, \n        on='target', \n        m=30\n    )\n    smooth_mappings.append(smooth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import HashingVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pickle as pkl\n\n# with open('name_vectorizer.pkl', 'wb') as f:\n#     pkl.dump(name_vectorizer, f)\n    \n# with open('brand_vectorizer.pkl', 'wb') as f:\n#     pkl.dump(brand_vectorizer, f)\n    \n# with open('rb_scaler.pkl', 'wb') as f:\n#     pkl.dump(rb_scaler, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"name_vectorizer = HashingVectorizer(n_features=10)\nname_vectorizer.fit(products_df.product_name)\nvecs_df = pd.DataFrame(name_vectorizer.transform(products_df.product_name).toarray())\nvecs_df.index = products_df.index\nvecs_df = vecs_df.add_prefix('product_name_')\nproducts_df = pd.concat([vecs_df, products_df], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"brand_vectorizer = HashingVectorizer(n_features=10)\nbrand_vectorizer.fit(products_df.brand_name)\nvecs_df = pd.DataFrame(brand_vectorizer.transform(products_df.brand_name).toarray())\nvecs_df.index = products_df.index\nvecs_df = vecs_df.add_prefix('brand_name_')\nproducts_df = pd.concat([vecs_df, products_df], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.drop(['product_name', 'brand_name'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.insert(0, 'price', products_df.pop('price'))\nproducts_df.insert(0, 'discount', products_df.pop('discount'))\nproducts_df.insert(0, 'user_id', products_df.pop('user_id'))\nproducts_df.insert(0, 'product_id', products_df.pop('product_id'))\nproducts_df.insert(0, 'order_id', products_df.pop('order_id'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"products_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = products_df.drop('target', axis=1).to_numpy()\ny = products_df['target'].to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.save('X.npy', X)\n# np.save('y.npy', y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rb_scaler = RobustScaler()\nX_train[:, :5] = rb_scaler.fit_transform(X_train[:, :5])\nX_test[:, :5] = rb_scaler.transform(X_test[:, :5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV, KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_reg = LogisticRegression(penalty='l2', max_iter=500, C=10000)\nlog_reg.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = log_reg.predict(X_test)\nprint(accuracy_score(y_pred, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, roc_curve\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_proba = log_reg.predict_proba(X_test)[:, 1]\nfpr, tpr, threshold = roc_curve(y_test, y_pred_proba)\nroc_auc = roc_auc_score(y_test, y_pred_proba)\n\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, 'b', label = 'AUC = %0.3f' % roc_auc)\nplt.legend(loc = 'lower right')\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0, 1])\nplt.ylim([0, 1])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_model = RandomForestClassifier(max_depth=15, n_estimators=50)\ntree_model.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = tree_model.predict(X_test)\nprint(accuracy_score(y_pred, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('rand_forest.pkl', 'wb') as f:\n    pkl.dump(tree_model, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_proba = tree_model.predict_proba(X_test)[:, 1]\nfpr, tpr, threshold = roc_curve(y_test, y_pred_proba)\nroc_auc = roc_auc_score(y_test, y_pred_proba)\n\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, 'b', label = 'AUC = %0.3f' % roc_auc)\nplt.legend(loc = 'lower right')\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0, 1])\nplt.ylim([0, 1])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_prod_df = pd.read_csv(Path('/kaggle/input', 'sbermarket-user-prod-data', 'user_prod_data.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_prod_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sort_basket(basket):\n    users = []\n    orders = []\n    prices = []\n    product_names = []\n    brand_names = []\n    master_category_ids = []\n    parent_category_ids = []\n    \n    for prod in basket:\n        users.append(user_prod_df.user_id[user_prod_df.product_id == prod].iloc[0])\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(Path('/kaggle/input/hack-sbermarket/dummy_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.iloc[0][1].split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_dummy_submission = sbermarket_tab_2_sample_submission[['user_id', 'product_id']].copy()\nmy_dummy_submission['product_id']  = my_dummy_submission['product_id'].apply(lambda s: str(s) + ' ')\nmy_dummy_submission = my_dummy_submission.groupby('user_id').product_id.sum().to_frame('product_id').reset_index()\nmy_dummy_submission.product_id = my_dummy_submission.product_id.apply(lambda s: s.split()[:50])\nmy_dummy_submission['product_id_added'] = my_dummy_submission.product_id\nsome_prod = my_dummy_submission.product_id_added.values[0]\nmy_dummy_submission.product_id_added = my_dummy_submission.product_id_added.apply(lambda s: s + some_prod[:50-len(s)])\ndummy_submission = my_dummy_submission[['user_id', 'product_id_added']]\ndummy_submission['Predicted'] = dummy_submission.product_id_added.apply(lambda s: ' '.join(prod for prod in s))\ndummy_submission['Id'] = dummy_submission.user_id\ndummy_submission = dummy_submission[['Id', 'Predicted']]\n# dummy_submission.to_csv('/kaggle/working/dummy_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Add orders to users"},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df = pd.read_csv(Path(general_folder, most_of_tab_subfolder, 'tab_1_orders.csv'))\norders_df = orders_df.drop_duplicates()\n# prods_df = pd.read_csv(Path(general_folder, 'sbermarket_tab_2_1/tab_2_products_2020-03-11.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df['order_created_time'] = pd.to_datetime(orders_df['order_created_time'], format='%Y-%m-%d %H:%M:%S')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df = orders_df.sort_values('order_created_time', ascending=False)\norders_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df = orders_df.drop_duplicates('order_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_latest_order_id(user_id, shift=0):\n    return orders_df[orders_df.user_id == user_id].order_id.iloc[shift]\n\n\ndef remove_dead_orders(prods_df):\n    return prods_df[prods_df.user_id.isin(orders_df.user_id)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df.iloc[120:125]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df = remove_dead_orders(prods_df)\nprods_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_df.drop('user_id', axis=1).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df = prods_df.merge(orders_df.drop('user_id', axis=1), on='order_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df.insert(0, 'product_id', prods_df.pop('product_id'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prods_from_order(order_id):\n    prods_in_order = prods_df[prods_df.order_id == order_id].order_id\n    return set(prods_in_order)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_time_df = orders_df[['order_id', 'order_created_time']]\n\n\norder2time = dict()\nfor i, row in orders_time_df.iterrows():\n    order2time[row.order_id] = row.order_created_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prev_order(order_id):\n#     cur_time = orders_time_df[orders_time_df.order_id == order_id].order_created_time.iloc[0]\n    cur_time = order2time[order_id]\n    prev_orders = orders_time_df[orders_time_df.order_created_time < cur_time].order_id\n    \n    if len(prev_orders) > 0:\n        prev_order_id = prev_orders.iloc[0]\n    else:\n        prev_order_id = np.nan\n        \n    return prev_order_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders_time_df[orders_time_df.order_id == 6951114]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"order_time_df = prods_df[['order_id', 'order_created_time']]\nprods_df['prev_order_id'] = order_time_df['order_id'].apply(get_prev_order)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test sample submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub_df = pd.read_csv(Path(general_folder, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub_df[sample_sub_df.Id.isin(users_df.user_id)].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prods_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub_df[sample_sub_df.Id.isin(prods_df.user_id)].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}