{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the part 1 of the complete end-to-end recommendation system.\nPart 2 coming soon.","metadata":{}},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport scipy.stats as ss\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nimport category_encoders as ce\nimport pickle\n\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-18T06:18:33.145202Z","iopub.execute_input":"2022-02-18T06:18:33.146107Z","iopub.status.idle":"2022-02-18T06:18:33.15214Z","shell.execute_reply.started":"2022-02-18T06:18:33.146063Z","shell.execute_reply":"2022-02-18T06:18:33.150787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic EDA and Data Cleaning","metadata":{}},{"cell_type":"code","source":"root_dir = '../input/h-and-m-personalized-fashion-recommendations'\narticles_df_raw = pd.read_csv(root_dir+'/articles.csv')\ncustomers_df_raw = pd.read_csv(root_dir+'/customers.csv')\ntransactions_df_raw = pd.read_csv(root_dir+'/transactions_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:18:33.154651Z","iopub.execute_input":"2022-02-18T06:18:33.155324Z","iopub.status.idle":"2022-02-18T06:19:53.461564Z","shell.execute_reply.started":"2022-02-18T06:18:33.155284Z","shell.execute_reply":"2022-02-18T06:19:53.4598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df = articles_df_raw.copy()\ncustomers_df = customers_df_raw.copy()\ntransactions_df = transactions_df_raw.copy()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:53.463778Z","iopub.execute_input":"2022-02-18T06:19:53.464074Z","iopub.status.idle":"2022-02-18T06:19:54.793448Z","shell.execute_reply.started":"2022-02-18T06:19:53.46404Z","shell.execute_reply":"2022-02-18T06:19:54.792442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploring articles ","metadata":{}},{"cell_type":"code","source":"articles_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:54.797008Z","iopub.execute_input":"2022-02-18T06:19:54.797363Z","iopub.status.idle":"2022-02-18T06:19:54.808794Z","shell.execute_reply.started":"2022-02-18T06:19:54.797316Z","shell.execute_reply":"2022-02-18T06:19:54.807403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.head().T","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:54.811341Z","iopub.execute_input":"2022-02-18T06:19:54.812206Z","iopub.status.idle":"2022-02-18T06:19:54.849102Z","shell.execute_reply.started":"2022-02-18T06:19:54.812158Z","shell.execute_reply":"2022-02-18T06:19:54.848194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:54.850787Z","iopub.execute_input":"2022-02-18T06:19:54.851311Z","iopub.status.idle":"2022-02-18T06:19:54.863178Z","shell.execute_reply.started":"2022-02-18T06:19:54.85126Z","shell.execute_reply":"2022-02-18T06:19:54.862316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting object dtype to categorical\ncategorical_columns = articles_df.select_dtypes(include='object').columns\nfor categorical_column in categorical_columns:\n    articles_df[categorical_column] = pd.Categorical(articles_df[categorical_column])","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:54.864745Z","iopub.execute_input":"2022-02-18T06:19:54.865452Z","iopub.status.idle":"2022-02-18T06:19:55.331928Z","shell.execute_reply.started":"2022-02-18T06:19:54.865375Z","shell.execute_reply":"2022-02-18T06:19:55.330934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_columns = articles_df.columns.values","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.333234Z","iopub.execute_input":"2022-02-18T06:19:55.333936Z","iopub.status.idle":"2022-02-18T06:19:55.337302Z","shell.execute_reply.started":"2022-02-18T06:19:55.3339Z","shell.execute_reply":"2022-02-18T06:19:55.336703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# percentage of uniqueness\nfor column in all_columns:\n    per = len(articles_df[column].unique()) / articles_df.shape[0] * 100.0\n    print(f'Percentage of unique {column}:\\t {per}%')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.338498Z","iopub.execute_input":"2022-02-18T06:19:55.338726Z","iopub.status.idle":"2022-02-18T06:19:55.389531Z","shell.execute_reply.started":"2022-02-18T06:19:55.3387Z","shell.execute_reply":"2022-02-18T06:19:55.388529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.390893Z","iopub.execute_input":"2022-02-18T06:19:55.391135Z","iopub.status.idle":"2022-02-18T06:19:55.411317Z","shell.execute_reply.started":"2022-02-18T06:19:55.391104Z","shell.execute_reply":"2022-02-18T06:19:55.410445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df['detail_desc'].isnull().sum() / articles_df.shape[0] * 100.0","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.412757Z","iopub.execute_input":"2022-02-18T06:19:55.413251Z","iopub.status.idle":"2022-02-18T06:19:55.420713Z","shell.execute_reply.started":"2022-02-18T06:19:55.413217Z","shell.execute_reply":"2022-02-18T06:19:55.41973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only 0.39 percent detail desc are nan. So drop them.","metadata":{}},{"cell_type":"code","source":"articles_df = articles_df.dropna().reset_index()\narticles_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.421928Z","iopub.execute_input":"2022-02-18T06:19:55.422177Z","iopub.status.idle":"2022-02-18T06:19:55.458215Z","shell.execute_reply.started":"2022-02-18T06:19:55.422147Z","shell.execute_reply":"2022-02-18T06:19:55.457117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 product names\nproduct_name = articles_df['prod_name'].value_counts()\nproduct_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.459379Z","iopub.execute_input":"2022-02-18T06:19:55.45965Z","iopub.status.idle":"2022-02-18T06:19:55.478344Z","shell.execute_reply.started":"2022-02-18T06:19:55.459618Z","shell.execute_reply":"2022-02-18T06:19:55.477744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 product types\nprod_type_name = articles_df['product_type_name'].value_counts()\nprod_type_name","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.482295Z","iopub.execute_input":"2022-02-18T06:19:55.482552Z","iopub.status.idle":"2022-02-18T06:19:55.498391Z","shell.execute_reply.started":"2022-02-18T06:19:55.482524Z","shell.execute_reply":"2022-02-18T06:19:55.497524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 product_group_name\nproduct_group_name = articles_df['product_group_name'].value_counts()\nproduct_group_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.499823Z","iopub.execute_input":"2022-02-18T06:19:55.500062Z","iopub.status.idle":"2022-02-18T06:19:55.51171Z","shell.execute_reply.started":"2022-02-18T06:19:55.500033Z","shell.execute_reply":"2022-02-18T06:19:55.510822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 graphical_appearance_name\ngraphical_appearance_name = articles_df['graphical_appearance_name'].value_counts()\ngraphical_appearance_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.513011Z","iopub.execute_input":"2022-02-18T06:19:55.513489Z","iopub.status.idle":"2022-02-18T06:19:55.528743Z","shell.execute_reply.started":"2022-02-18T06:19:55.513437Z","shell.execute_reply":"2022-02-18T06:19:55.527915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 colour_group_name\ncolour_group_name = articles_df['colour_group_name'].value_counts()\ncolour_group_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.530454Z","iopub.execute_input":"2022-02-18T06:19:55.531446Z","iopub.status.idle":"2022-02-18T06:19:55.541841Z","shell.execute_reply.started":"2022-02-18T06:19:55.531375Z","shell.execute_reply":"2022-02-18T06:19:55.540911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 perceived_colour_value_name\nperceived_colour_value_name = articles_df['perceived_colour_value_name'].value_counts()\nperceived_colour_value_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.543015Z","iopub.execute_input":"2022-02-18T06:19:55.543697Z","iopub.status.idle":"2022-02-18T06:19:55.555229Z","shell.execute_reply.started":"2022-02-18T06:19:55.543661Z","shell.execute_reply":"2022-02-18T06:19:55.554284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 perceived_colour_master_name\nperceived_colour_master_name = articles_df['perceived_colour_master_name'].value_counts()\nperceived_colour_master_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.55677Z","iopub.execute_input":"2022-02-18T06:19:55.557105Z","iopub.status.idle":"2022-02-18T06:19:55.56899Z","shell.execute_reply.started":"2022-02-18T06:19:55.557077Z","shell.execute_reply":"2022-02-18T06:19:55.568206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 department names\ndepartment_name = articles_df['department_name'].value_counts()\ndepartment_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.570023Z","iopub.execute_input":"2022-02-18T06:19:55.570648Z","iopub.status.idle":"2022-02-18T06:19:55.58428Z","shell.execute_reply.started":"2022-02-18T06:19:55.570611Z","shell.execute_reply":"2022-02-18T06:19:55.583465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 index_name\nindex_name = articles_df['index_name'].value_counts()\nindex_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.585826Z","iopub.execute_input":"2022-02-18T06:19:55.586155Z","iopub.status.idle":"2022-02-18T06:19:55.598655Z","shell.execute_reply.started":"2022-02-18T06:19:55.586113Z","shell.execute_reply":"2022-02-18T06:19:55.597758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 section_name\nsection_name = articles_df['section_name'].value_counts()\nsection_name.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.600094Z","iopub.execute_input":"2022-02-18T06:19:55.600582Z","iopub.status.idle":"2022-02-18T06:19:55.611053Z","shell.execute_reply.started":"2022-02-18T06:19:55.600541Z","shell.execute_reply":"2022-02-18T06:19:55.610252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 10 detail_desc\ndetail_desc = articles_df['detail_desc'].value_counts()\ndetail_desc.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.61222Z","iopub.execute_input":"2022-02-18T06:19:55.612622Z","iopub.status.idle":"2022-02-18T06:19:55.634884Z","shell.execute_reply.started":"2022-02-18T06:19:55.612577Z","shell.execute_reply":"2022-02-18T06:19:55.633894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='perceived_colour_value_name',data=articles_df, hue='perceived_colour_master_name')\nplt.legend(bbox_to_anchor=(1.1, 1))\nplt.xticks(rotation=90)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:55.636348Z","iopub.execute_input":"2022-02-18T06:19:55.63783Z","iopub.status.idle":"2022-02-18T06:19:56.625772Z","shell.execute_reply.started":"2022-02-18T06:19:55.637782Z","shell.execute_reply":"2022-02-18T06:19:56.624958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='index_group_name',data=articles_df, hue='index_name')\nplt.legend(bbox_to_anchor=(1.1, 1))","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:56.626778Z","iopub.execute_input":"2022-02-18T06:19:56.627454Z","iopub.status.idle":"2022-02-18T06:19:57.088429Z","shell.execute_reply.started":"2022-02-18T06:19:56.62739Z","shell.execute_reply":"2022-02-18T06:19:57.087499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Computing correlation between two categorical features","metadata":{}},{"cell_type":"code","source":"# function courtesy - \n# https://stackoverflow.com/questions/46498455/categorical-features-correlation/46498792#46498792\ndef cramers_v(confusion_matrix):\n    \"\"\" calculate Cramers V statistic for categorial-categorial association.\n        uses correction from Bergsma and Wicher,\n        Journal of the Korean Statistical Society 42 (2013): 323-328\n    \"\"\"\n    chi2 = ss.chi2_contingency(confusion_matrix)[0]\n    n = confusion_matrix.sum()\n    phi2 = chi2 / n\n    r, k = confusion_matrix.shape\n    phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))\n    rcorr = r - ((r-1)**2)/(n-1)\n    kcorr = k - ((k-1)**2)/(n-1)\n    return np.sqrt(phi2corr / min((kcorr-1), (rcorr-1)))","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:57.089702Z","iopub.execute_input":"2022-02-18T06:19:57.089932Z","iopub.status.idle":"2022-02-18T06:19:57.09818Z","shell.execute_reply.started":"2022-02-18T06:19:57.089906Z","shell.execute_reply":"2022-02-18T06:19:57.097218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = 'department_name'\nf2 = 'garment_group_name'\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f2])\ncramers_v(confusion_matrix.values)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:57.09999Z","iopub.execute_input":"2022-02-18T06:19:57.100233Z","iopub.status.idle":"2022-02-18T06:19:57.384178Z","shell.execute_reply.started":"2022-02-18T06:19:57.100207Z","shell.execute_reply":"2022-02-18T06:19:57.383274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like there is a high correlation between **department_name** and **garment_group_name**.\nThese can be combined together.","metadata":{}},{"cell_type":"code","source":"f1 = 'index_name'\nf2 = 'index_group_name'\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f2])\ncramers_v(confusion_matrix.values)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:57.385581Z","iopub.execute_input":"2022-02-18T06:19:57.385804Z","iopub.status.idle":"2022-02-18T06:19:57.415048Z","shell.execute_reply.started":"2022-02-18T06:19:57.385777Z","shell.execute_reply":"2022-02-18T06:19:57.414344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"High correlation again\n\n**index_name** and **index_group_name** can also be combined.","metadata":{}},{"cell_type":"markdown","source":"Combine these as well.","metadata":{}},{"cell_type":"code","source":"f1 = 'colour_group_name'\nf2 = 'perceived_colour_value_name'\nf3 = 'perceived_colour_master_name'\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f2])\nf1_f2 = cramers_v(confusion_matrix.values)\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f3])\nf1_f3 = cramers_v(confusion_matrix.values)\n\nconfusion_matrix = pd.crosstab(articles_df[f2], articles_df[f3])\nf2_f3 = cramers_v(confusion_matrix.values)\n\nprint(f'Between f1 and f2: {f1_f2}')\nprint(f'Between f1 and f3: {f1_f3}')\nprint(f'Between f2 and f3: {f2_f3}')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:57.416168Z","iopub.execute_input":"2022-02-18T06:19:57.416403Z","iopub.status.idle":"2022-02-18T06:19:57.498395Z","shell.execute_reply.started":"2022-02-18T06:19:57.416377Z","shell.execute_reply":"2022-02-18T06:19:57.497231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Combine all three.","metadata":{}},{"cell_type":"code","source":"f1 = 'prod_name'\nf2 = 'product_type_name'\nf3 = 'product_group_name'\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f2])\nf1_f2 = cramers_v(confusion_matrix.values)\n\nconfusion_matrix = pd.crosstab(articles_df[f1], articles_df[f3])\nf1_f3 = cramers_v(confusion_matrix.values)\n\nconfusion_matrix = pd.crosstab(articles_df[f2], articles_df[f3])\nf2_f3 = cramers_v(confusion_matrix.values)\n\nprint(f'Between f1 and f2: {f1_f2}')\nprint(f'Between f1 and f3: {f1_f3}')\nprint(f'Between f2 and f3: {f2_f3}')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:19:57.499925Z","iopub.execute_input":"2022-02-18T06:19:57.500232Z","iopub.status.idle":"2022-02-18T06:20:12.684765Z","shell.execute_reply.started":"2022-02-18T06:19:57.500197Z","shell.execute_reply":"2022-02-18T06:20:12.683583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Combine all three. \n\nNote: We combine only if the correlation value is greater than 0.5","metadata":{}},{"cell_type":"markdown","source":"### Exploring customers","metadata":{}},{"cell_type":"code","source":"customers_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:12.686268Z","iopub.execute_input":"2022-02-18T06:20:12.686688Z","iopub.status.idle":"2022-02-18T06:20:12.69335Z","shell.execute_reply.started":"2022-02-18T06:20:12.686642Z","shell.execute_reply":"2022-02-18T06:20:12.692376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.head(2).T","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:12.694996Z","iopub.execute_input":"2022-02-18T06:20:12.69524Z","iopub.status.idle":"2022-02-18T06:20:12.718692Z","shell.execute_reply.started":"2022-02-18T06:20:12.695204Z","shell.execute_reply":"2022-02-18T06:20:12.71768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:12.720311Z","iopub.execute_input":"2022-02-18T06:20:12.720642Z","iopub.status.idle":"2022-02-18T06:20:12.73139Z","shell.execute_reply.started":"2022-02-18T06:20:12.720595Z","shell.execute_reply":"2022-02-18T06:20:12.730745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting object dtype to categorical\ncategorical_columns = customers_df.select_dtypes(include='object').columns\nfor categorical_column in categorical_columns:\n    if 'customer_id' not in categorical_column:\n        customers_df[categorical_column] = pd.Categorical(customers_df[categorical_column])","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:12.732762Z","iopub.execute_input":"2022-02-18T06:20:12.733331Z","iopub.status.idle":"2022-02-18T06:20:15.137406Z","shell.execute_reply.started":"2022-02-18T06:20:12.733296Z","shell.execute_reply":"2022-02-18T06:20:15.136647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"100.0 * customers_df.isnull().sum() / customers_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:15.138909Z","iopub.execute_input":"2022-02-18T06:20:15.139394Z","iopub.status.idle":"2022-02-18T06:20:15.328113Z","shell.execute_reply.started":"2022-02-18T06:20:15.139358Z","shell.execute_reply":"2022-02-18T06:20:15.327476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**FN** and **Active** has more than 30% Nans. Drop them.\n\nAlso drop postal code (for this version, later we will analyze it).","metadata":{}},{"cell_type":"code","source":"customers_df.drop(['FN', 'Active', 'postal_code'], inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:15.329434Z","iopub.execute_input":"2022-02-18T06:20:15.329876Z","iopub.status.idle":"2022-02-18T06:20:15.392996Z","shell.execute_reply.started":"2022-02-18T06:20:15.329844Z","shell.execute_reply":"2022-02-18T06:20:15.392313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:15.394363Z","iopub.execute_input":"2022-02-18T06:20:15.394835Z","iopub.status.idle":"2022-02-18T06:20:15.574357Z","shell.execute_reply.started":"2022-02-18T06:20:15.394801Z","shell.execute_reply":"2022-02-18T06:20:15.573378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We still have NaNs. Let's fill them.","metadata":{}},{"cell_type":"markdown","source":"Let's plot the distributions first","metadata":{}},{"cell_type":"code","source":"sns.countplot(customers_df.club_member_status)\ncustomers_df.club_member_status.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:15.575945Z","iopub.execute_input":"2022-02-18T06:20:15.576182Z","iopub.status.idle":"2022-02-18T06:20:15.814772Z","shell.execute_reply.started":"2022-02-18T06:20:15.576153Z","shell.execute_reply":"2022-02-18T06:20:15.813783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(customers_df.fashion_news_frequency)\ncustomers_df.fashion_news_frequency.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:15.816011Z","iopub.execute_input":"2022-02-18T06:20:15.816261Z","iopub.status.idle":"2022-02-18T06:20:16.048946Z","shell.execute_reply.started":"2022-02-18T06:20:15.816228Z","shell.execute_reply":"2022-02-18T06:20:16.047888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(customers_df.age)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:16.050204Z","iopub.execute_input":"2022-02-18T06:20:16.050476Z","iopub.status.idle":"2022-02-18T06:20:21.533564Z","shell.execute_reply.started":"2022-02-18T06:20:16.050445Z","shell.execute_reply":"2022-02-18T06:20:21.532645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='club_member_status', y='age', data=customers_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:20:21.540486Z","iopub.execute_input":"2022-02-18T06:20:21.541046Z","iopub.status.idle":"2022-02-18T06:21:01.774394Z","shell.execute_reply.started":"2022-02-18T06:20:21.541004Z","shell.execute_reply":"2022-02-18T06:21:01.773456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df_ = customers_df.copy()\nmap_means = customers_df_.groupby('club_member_status')['age'].mean().to_dict()\nmap_means","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:21:01.775855Z","iopub.execute_input":"2022-02-18T06:21:01.776109Z","iopub.status.idle":"2022-02-18T06:21:01.824863Z","shell.execute_reply.started":"2022-02-18T06:21:01.776076Z","shell.execute_reply":"2022-02-18T06:21:01.82401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute club_member_status by majority vote value which is \"Active\".\ncustomers_df['club_member_status'].fillna('ACTIVE', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:21:01.826344Z","iopub.execute_input":"2022-02-18T06:21:01.826609Z","iopub.status.idle":"2022-02-18T06:21:01.833092Z","shell.execute_reply.started":"2022-02-18T06:21:01.82658Z","shell.execute_reply":"2022-02-18T06:21:01.832197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute fashion_news_frequency by majority vote value which is \"NONE\".\ncustomers_df['fashion_news_frequency'].fillna('NONE', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:21:01.834588Z","iopub.execute_input":"2022-02-18T06:21:01.835115Z","iopub.status.idle":"2022-02-18T06:21:01.852155Z","shell.execute_reply.started":"2022-02-18T06:21:01.835045Z","shell.execute_reply":"2022-02-18T06:21:01.851395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute age based on club_member_status\n\n# Using for loop here, instead of direct mapping, since direct mapping seemed to break the code. \n\n# idx_nan_age = customers_df.loc[np.isnan(df['age'])].index\n# customers_df.loc[idx_nan_age,'age'].loc[idx_nan_age] = customers_df['club_member_status'].loc[idx_nan_age].map(map_means)\n\n# Suggestions are always welcome.\n\nmask = customers_df['age'].isnull()\nage_values = customers_df.loc[mask, 'club_member_status'].map(map_means).values\nis_nan_age = customers_df.loc[mask, 'age'].index.values\n\nfor i in tqdm(range(len(is_nan_age))):\n    customers_df.loc[is_nan_age[i], 'age'] = age_values[i]","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:21:01.853582Z","iopub.execute_input":"2022-02-18T06:21:01.854075Z","iopub.status.idle":"2022-02-18T06:23:06.193526Z","shell.execute_reply.started":"2022-02-18T06:21:01.854037Z","shell.execute_reply":"2022-02-18T06:23:06.192538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.194836Z","iopub.execute_input":"2022-02-18T06:23:06.195081Z","iopub.status.idle":"2022-02-18T06:23:06.378842Z","shell.execute_reply.started":"2022-02-18T06:23:06.195052Z","shell.execute_reply":"2022-02-18T06:23:06.377691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.fashion_news_frequency.unique()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.380188Z","iopub.execute_input":"2022-02-18T06:23:06.380402Z","iopub.status.idle":"2022-02-18T06:23:06.399047Z","shell.execute_reply.started":"2022-02-18T06:23:06.380376Z","shell.execute_reply":"2022-02-18T06:23:06.398153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looking at the unique values of **fashion_news_frequency**, we see there is **NONE** and **None**.\n\nAre they same? Maybe yes. Unless **NONE** is an acronym for something.\n\nFor now let's consider them to be same.","metadata":{}},{"cell_type":"code","source":"customers_df.fashion_news_frequency.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.400482Z","iopub.execute_input":"2022-02-18T06:23:06.400747Z","iopub.status.idle":"2022-02-18T06:23:06.415684Z","shell.execute_reply.started":"2022-02-18T06:23:06.400717Z","shell.execute_reply":"2022-02-18T06:23:06.414732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = customers_df[customers_df['fashion_news_frequency'] == 'None'].index.values\ncustomers_df.loc[mask, 'fashion_news_frequency'] = 'NONE'","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.417362Z","iopub.execute_input":"2022-02-18T06:23:06.417651Z","iopub.status.idle":"2022-02-18T06:23:06.427242Z","shell.execute_reply.started":"2022-02-18T06:23:06.417615Z","shell.execute_reply":"2022-02-18T06:23:06.426327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.fashion_news_frequency.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.428697Z","iopub.execute_input":"2022-02-18T06:23:06.429441Z","iopub.status.idle":"2022-02-18T06:23:06.44658Z","shell.execute_reply.started":"2022-02-18T06:23:06.429267Z","shell.execute_reply":"2022-02-18T06:23:06.445393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explore transactions","metadata":{}},{"cell_type":"code","source":"transactions_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.448199Z","iopub.execute_input":"2022-02-18T06:23:06.448541Z","iopub.status.idle":"2022-02-18T06:23:06.455648Z","shell.execute_reply.started":"2022-02-18T06:23:06.448495Z","shell.execute_reply":"2022-02-18T06:23:06.454547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.457149Z","iopub.execute_input":"2022-02-18T06:23:06.457894Z","iopub.status.idle":"2022-02-18T06:23:06.473972Z","shell.execute_reply.started":"2022-02-18T06:23:06.457843Z","shell.execute_reply":"2022-02-18T06:23:06.472729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df['t_dat'] = pd.to_datetime(transactions_df['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:06.477061Z","iopub.execute_input":"2022-02-18T06:23:06.47732Z","iopub.status.idle":"2022-02-18T06:23:12.820031Z","shell.execute_reply.started":"2022-02-18T06:23:06.477291Z","shell.execute_reply":"2022-02-18T06:23:12.819274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:12.821012Z","iopub.execute_input":"2022-02-18T06:23:12.821829Z","iopub.status.idle":"2022-02-18T06:23:12.830855Z","shell.execute_reply.started":"2022-02-18T06:23:12.821764Z","shell.execute_reply":"2022-02-18T06:23:12.829813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:12.832354Z","iopub.execute_input":"2022-02-18T06:23:12.832735Z","iopub.status.idle":"2022-02-18T06:23:16.855191Z","shell.execute_reply.started":"2022-02-18T06:23:12.832695Z","shell.execute_reply":"2022-02-18T06:23:16.854366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:16.856595Z","iopub.execute_input":"2022-02-18T06:23:16.856894Z","iopub.status.idle":"2022-02-18T06:23:26.318227Z","shell.execute_reply.started":"2022-02-18T06:23:16.85685Z","shell.execute_reply":"2022-02-18T06:23:26.317197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_df = transactions_df[transactions_df.duplicated()]\ndup_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:23:26.319617Z","iopub.execute_input":"2022-02-18T06:23:26.319868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 15\ntransactions_df[(transactions_df.t_dat == dup_df.loc[idx].t_dat) & \n                (transactions_df.customer_id == dup_df.loc[idx].customer_id) & \n                (transactions_df.article_id == dup_df.loc[idx].article_id) & \n                (transactions_df.price == dup_df.loc[idx].price) & \n                (transactions_df.sales_channel_id == dup_df.loc[idx].sales_channel_id)\n               ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now club the duplicate rows and mention the count of duplicate rows into a new column **article_purchase_count**","metadata":{}},{"cell_type":"code","source":"%%time\ntransactions_df = transactions_df.groupby(transactions_df.columns.tolist()).size().reset_index().rename(columns={0:'article_purchase_count'})\ntransactions_df.head()","metadata":{"execution":{"iopub.status.idle":"2022-02-18T06:24:33.88636Z","shell.execute_reply.started":"2022-02-18T06:23:52.118933Z","shell.execute_reply":"2022-02-18T06:24:33.885491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 20\ntransactions_df[(transactions_df.customer_id == dup_df.loc[idx].customer_id) & \n                (transactions_df.article_id == dup_df.loc[idx].article_id)]","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:33.887677Z","iopub.execute_input":"2022-02-18T06:24:33.887918Z","iopub.status.idle":"2022-02-18T06:24:39.578835Z","shell.execute_reply.started":"2022-02-18T06:24:33.887889Z","shell.execute_reply":"2022-02-18T06:24:39.577948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see the purchasing history of a single customer.","metadata":{}},{"cell_type":"code","source":"customer_0_df = transactions_df[transactions_df.customer_id == transactions_df.iloc[0,1]]\ncustomer_0_df","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:39.580384Z","iopub.execute_input":"2022-02-18T06:24:39.583217Z","iopub.status.idle":"2022-02-18T06:24:45.138365Z","shell.execute_reply.started":"2022-02-18T06:24:39.583176Z","shell.execute_reply":"2022-02-18T06:24:45.137728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"### Customers FE","metadata":{}},{"cell_type":"code","source":"customers_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.139728Z","iopub.execute_input":"2022-02-18T06:24:45.139954Z","iopub.status.idle":"2022-02-18T06:24:45.153811Z","shell.execute_reply.started":"2022-02-18T06:24:45.139927Z","shell.execute_reply":"2022-02-18T06:24:45.15292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.15504Z","iopub.execute_input":"2022-02-18T06:24:45.155271Z","iopub.status.idle":"2022-02-18T06:24:45.172678Z","shell.execute_reply.started":"2022-02-18T06:24:45.155243Z","shell.execute_reply":"2022-02-18T06:24:45.17152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummies = pd.get_dummies(customers_df[['club_member_status', 'fashion_news_frequency']], drop_first=True)\ncustomers_df = pd.concat([customers_df, dummies], axis=1)\ncustomers_df.drop(['club_member_status','fashion_news_frequency'], axis=1, inplace=True)\ncustomers_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.173713Z","iopub.execute_input":"2022-02-18T06:24:45.173948Z","iopub.status.idle":"2022-02-18T06:24:45.352321Z","shell.execute_reply.started":"2022-02-18T06:24:45.173921Z","shell.execute_reply":"2022-02-18T06:24:45.35124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rescale age data\nage_scaler = MinMaxScaler()\n\ncustomers_df[['age']] = age_scaler.fit_transform(customers_df[['age']])\ncustomers_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.353953Z","iopub.execute_input":"2022-02-18T06:24:45.354209Z","iopub.status.idle":"2022-02-18T06:24:45.393993Z","shell.execute_reply.started":"2022-02-18T06:24:45.35418Z","shell.execute_reply":"2022-02-18T06:24:45.392744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Articles FE","metadata":{}},{"cell_type":"code","source":"articles_df.head().T","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.395712Z","iopub.execute_input":"2022-02-18T06:24:45.396073Z","iopub.status.idle":"2022-02-18T06:24:45.494223Z","shell.execute_reply.started":"2022-02-18T06:24:45.396025Z","shell.execute_reply":"2022-02-18T06:24:45.493185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_interesting_columns = ['index', 'product_code', 'product_type_no', 'graphical_appearance_no',\n                           'colour_group_code', 'perceived_colour_value_id', 'perceived_colour_master_id',\n                           'department_no', 'index_code', 'index_group_no', 'section_no', 'garment_group_no',\n                           ]\n\narticles_df.drop(non_interesting_columns, axis=1, inplace=True)\narticles_df.head().T","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.496281Z","iopub.execute_input":"2022-02-18T06:24:45.496648Z","iopub.status.idle":"2022-02-18T06:24:45.523175Z","shell.execute_reply.started":"2022-02-18T06:24:45.496601Z","shell.execute_reply":"2022-02-18T06:24:45.522264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge department_name and garment_group_name\n\nf1 = 'department_name'\nf2 = 'garment_group_name'\n\narticles_df['department'] = articles_df[f1].astype(str) + '_' +articles_df[f2].astype(str)\narticles_df.drop([f1,f2], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.524996Z","iopub.execute_input":"2022-02-18T06:24:45.525698Z","iopub.status.idle":"2022-02-18T06:24:45.63044Z","shell.execute_reply.started":"2022-02-18T06:24:45.525629Z","shell.execute_reply":"2022-02-18T06:24:45.629432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge index_name and index_group_name\n\nf1 = 'index_name'\nf2 = 'index_group_name'\n\narticles_df['index'] = articles_df[f1].astype(str) + '_' + articles_df[f2].astype(str)\narticles_df.drop([f1,f2], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.631695Z","iopub.execute_input":"2022-02-18T06:24:45.632135Z","iopub.status.idle":"2022-02-18T06:24:45.730354Z","shell.execute_reply.started":"2022-02-18T06:24:45.632104Z","shell.execute_reply":"2022-02-18T06:24:45.729278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = 'colour_group_name'\nf2 = 'perceived_colour_value_name'\nf3 = 'perceived_colour_master_name'\n\narticles_df['color'] = articles_df[f1].astype(str) + '_' + articles_df[f2].astype(str) + '_' + articles_df[f3].astype(str)\narticles_df.drop([f1,f2,f3], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.731593Z","iopub.execute_input":"2022-02-18T06:24:45.731993Z","iopub.status.idle":"2022-02-18T06:24:45.89791Z","shell.execute_reply.started":"2022-02-18T06:24:45.731959Z","shell.execute_reply":"2022-02-18T06:24:45.896832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = 'prod_name'\nf2 = 'product_type_name'\nf3 = 'product_group_name'\n\narticles_df['product'] = articles_df[f1].astype(str) + '_' +  articles_df[f2].astype(str) + '_' + articles_df[f3].astype(str)\narticles_df.drop([f1,f2,f3], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:45.899207Z","iopub.execute_input":"2022-02-18T06:24:45.899457Z","iopub.status.idle":"2022-02-18T06:24:46.089459Z","shell.execute_reply.started":"2022-02-18T06:24:45.899402Z","shell.execute_reply":"2022-02-18T06:24:46.088707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.head().T","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:46.090684Z","iopub.execute_input":"2022-02-18T06:24:46.091057Z","iopub.status.idle":"2022-02-18T06:24:46.105712Z","shell.execute_reply.started":"2022-02-18T06:24:46.091027Z","shell.execute_reply":"2022-02-18T06:24:46.104621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:46.106997Z","iopub.execute_input":"2022-02-18T06:24:46.107655Z","iopub.status.idle":"2022-02-18T06:24:46.196905Z","shell.execute_reply.started":"2022-02-18T06:24:46.107614Z","shell.execute_reply":"2022-02-18T06:24:46.19591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hash encoding categorical columns of articles_df\n\nencoder = ce.HashingEncoder(cols=['graphical_appearance_name',\n                                  'section_name',\n                                  'detail_desc',\n                                  'department',\n                                  'index',\n                                  'color',\n                                  'product'\n                                 ], n_components=1000)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:46.19845Z","iopub.execute_input":"2022-02-18T06:24:46.199299Z","iopub.status.idle":"2022-02-18T06:24:46.205613Z","shell.execute_reply.started":"2022-02-18T06:24:46.199252Z","shell.execute_reply":"2022-02-18T06:24:46.204516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df = encoder.fit_transform(articles_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:24:46.207264Z","iopub.execute_input":"2022-02-18T06:24:46.207558Z","iopub.status.idle":"2022-02-18T06:27:03.314435Z","shell.execute_reply.started":"2022-02-18T06:24:46.207524Z","shell.execute_reply":"2022-02-18T06:27:03.312943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:27:03.316347Z","iopub.execute_input":"2022-02-18T06:27:03.316679Z","iopub.status.idle":"2022-02-18T06:27:03.325116Z","shell.execute_reply.started":"2022-02-18T06:27:03.316643Z","shell.execute_reply":"2022-02-18T06:27:03.32422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transactions FE","metadata":{}},{"cell_type":"code","source":"transactions_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:27:03.326832Z","iopub.execute_input":"2022-02-18T06:27:03.327518Z","iopub.status.idle":"2022-02-18T06:27:03.351508Z","shell.execute_reply.started":"2022-02-18T06:27:03.327465Z","shell.execute_reply":"2022-02-18T06:27:03.350522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rescale \napc_scaler = StandardScaler()\n\ntransactions_df[['price', 'article_purchase_count']] = apc_scaler.fit_transform(transactions_df[['price', 'article_purchase_count']])\ntransactions_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:27:03.353411Z","iopub.execute_input":"2022-02-18T06:27:03.354025Z","iopub.status.idle":"2022-02-18T06:27:06.136266Z","shell.execute_reply.started":"2022-02-18T06:27:03.353973Z","shell.execute_reply":"2022-02-18T06:27:06.135563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save as a pickle file\ntransactions_df.to_pickle('t_df.pkl')\narticles_df.to_pickle('a_df.pkl')\ncustomers_df.to_pickle('c_df.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T06:27:06.137634Z","iopub.execute_input":"2022-02-18T06:27:06.138129Z","iopub.status.idle":"2022-02-18T06:27:17.278247Z","shell.execute_reply.started":"2022-02-18T06:27:06.138096Z","shell.execute_reply":"2022-02-18T06:27:17.276992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Done!')","metadata":{"execution":{"iopub.status.busy":"2022-02-18T07:20:31.529665Z","iopub.execute_input":"2022-02-18T07:20:31.530002Z","iopub.status.idle":"2022-02-18T07:20:31.557409Z","shell.execute_reply.started":"2022-02-18T07:20:31.529922Z","shell.execute_reply":"2022-02-18T07:20:31.556499Z"},"trusted":true},"execution_count":null,"outputs":[]}]}