{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### EDA & Business Understanding\n\nThe first step in any robust machine learning project is a deep and thorough understanding of the data. In this phase, we will conduct a Exploratory Data Analysis (EDA) to understand our customers, articles, and transaction patterns.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom collections import Counter\nimport gc\nimport warnings\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n\nfrom scipy.stats import skew, pearsonr\nfrom wordcloud import WordCloud\n\nsns.set_style('whitegrid')\nplt.style.use('fivethirtyeight')\nsns.set_palette('husl')\n\nwarnings.filterwarnings(\"ignore\", category=pd.errors.SettingWithCopyWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:14.315509Z","iopub.execute_input":"2025-09-23T20:20:14.315823Z","iopub.status.idle":"2025-09-23T20:20:18.698675Z","shell.execute_reply.started":"2025-09-23T20:20:14.315798Z","shell.execute_reply":"2025-09-23T20:20:18.697627Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\narticles = pl.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pl.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\n\nlazy_transactions = pl.scan_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:18.700093Z","iopub.execute_input":"2025-09-23T20:20:18.700644Z","iopub.status.idle":"2025-09-23T20:20:21.940483Z","shell.execute_reply.started":"2025-09-23T20:20:18.700614Z","shell.execute_reply":"2025-09-23T20:20:21.939408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customers.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:21.941375Z","iopub.execute_input":"2025-09-23T20:20:21.941780Z","iopub.status.idle":"2025-09-23T20:20:21.970618Z","shell.execute_reply.started":"2025-09-23T20:20:21.941754Z","shell.execute_reply":"2025-09-23T20:20:21.969706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:21.972438Z","iopub.execute_input":"2025-09-23T20:20:21.972738Z","iopub.status.idle":"2025-09-23T20:20:21.981678Z","shell.execute_reply.started":"2025-09-23T20:20:21.972715Z","shell.execute_reply":"2025-09-23T20:20:21.980769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"DATASET OVERVIEW\")\n\nprint(f\"Articles : {articles.shape}\")\nprint(f\"Customers : {customers.shape}\")\n\ntransactions_schema = lazy_transactions.schema\ntransactions_count = lazy_transactions.select(pl.len()).collect().item(0, 0)\nprint(f\"Transactions: {transactions_count:,} rows × {len(transactions_schema)} cols\")\n\nprint(\"\\nARTICLES DATASET:\")\nprint(\"Column Name\".ljust(25) + \"Data Type\".ljust(15) + \"Non-null Count\")\nprint(\"-\" * 55)\nfor col, dtype in zip(articles.columns, articles.dtypes):\n    non_null = articles.select(pl.col(col).is_not_null().sum()).item(0, 0)\n    print(f\"{col:<25} {str(dtype):<15} {non_null:>10,}\")\n\nprint(\"\\CUSTOMERS DATASET:\")\nprint(\"Column Name\".ljust(25) + \"Data Type\".ljust(15) + \"Non-null Count\")\nprint(\"-\" * 55)\nfor col, dtype in zip(customers.columns, customers.dtypes):\n    non_null = customers.select(pl.col(col).is_not_null().sum()).item(0, 0)\n    print(f\"{col:<25} {str(dtype):<15} {non_null:>10,}\")\n\nprint(\"\\nTRANSACTIONS DATASET:\")\nprint(\"Column Name\".ljust(25) + \"Data Type\".ljust(15) + \"Sample Values\")\nprint(\"-\" * 55)\ntransactions_sample = lazy_transactions.head(3).collect()\nfor col, dtype in transactions_schema.items():\n    sample_vals = transactions_sample.select(pl.col(col)).to_series().to_list()\n    print(f\"{col:<25} {str(dtype):<15} {str(sample_vals[:2])}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:21.982739Z","iopub.execute_input":"2025-09-23T20:20:21.983106Z","iopub.status.idle":"2025-09-23T20:20:33.281687Z","shell.execute_reply.started":"2025-09-23T20:20:21.983071Z","shell.execute_reply":"2025-09-23T20:20:33.280256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"MISSING VALUES:\")\n\nprint(\"ARTICLES:\")\narticles_missing = (\n    articles\n    .select([\n        (pl.col(col).is_null().sum() / len(articles) * 100).alias(f\"{col}_missing_pct\")\n        for col in articles.columns\n    ])\n    .unpivot(variable_name=\"column\", value_name=\"missing_percentage\")\n    .filter(pl.col(\"missing_percentage\") > 0)\n    .sort(\"missing_percentage\", descending=True)\n)\n\nif len(articles_missing) > 0:\n    print(articles_missing.to_pandas())\nelse:\n    print(\"No missing values found in articles dataset\")\n\nprint(\"CUSTOMERS:\")\ncustomers_missing = (\n    customers\n    .select([\n        (pl.col(col).is_null().sum() / len(customers) * 100).alias(f\"{col}_missing_pct\")\n        for col in customers.columns\n    ])\n    .unpivot(variable_name=\"column\", value_name=\"missing_percentage\")\n    .filter(pl.col(\"missing_percentage\") > 0)\n    .sort(\"missing_percentage\", descending=True)\n)\n\nif len(customers_missing) > 0:\n    print(customers_missing.to_pandas())\nelse:\n    print(\"No missing values found in customers dataset\")\n\nprint(\"TRANSACTIONS - Missing Value Summary:\")\ntransactions_missing = (\n    lazy_transactions\n    .select([\n        (pl.col(col).is_null().sum()).alias(f\"{col}_nulls\")\n        for col in lazy_transactions.columns\n    ])\n    .collect()\n    .transpose(include_header=True, header_name=\"column\")\n    .rename({\"column_0\": \"null_count\"})\n    .with_columns([\n        (pl.col(\"null_count\") / transactions_count * 100).alias(\"missing_percentage\")\n    ])\n    .filter(pl.col(\"missing_percentage\") > 0)\n    .sort(\"missing_percentage\", descending=True)\n)\n\nif len(transactions_missing) > 0:\n    print(transactions_missing.to_pandas())\nelse:\n    print(\"No missing values found in transactions dataset\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:33.282782Z","iopub.execute_input":"2025-09-23T20:20:33.283185Z","iopub.status.idle":"2025-09-23T20:20:38.793897Z","shell.execute_reply.started":"2025-09-23T20:20:33.283157Z","shell.execute_reply":"2025-09-23T20:20:38.792761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"MEMORY USAGE\")\n\narticles_memory = articles.estimated_size(\"mb\")\ncustomers_memory = customers.estimated_size(\"mb\") \n\nprint(f\"Articles dataset:    {articles_memory:.1f} MB\")\nprint(f\"Customers dataset:   {customers_memory:.1f} MB\")\nprint(f\"Transactions dataset: ~{transactions_count * len(transactions_schema) * 8 / 1024 / 1024:.0f} MB (estimated)\")\nprint(f\"TOTAL ESTIMATED:     ~{articles_memory + customers_memory + (transactions_count * len(transactions_schema) * 8 / 1024 / 1024):.0f} MB\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:38.795829Z","iopub.execute_input":"2025-09-23T20:20:38.796278Z","iopub.status.idle":"2025-09-23T20:20:38.805064Z","shell.execute_reply.started":"2025-09-23T20:20:38.796240Z","shell.execute_reply":"2025-09-23T20:20:38.803873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"TEMPORAL COVERAGE:\")\n\ndate_info = (\n    lazy_transactions\n    .select([\n        pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\").alias(\"date\")\n    ])\n    .select([\n        pl.col(\"date\").min().alias(\"earliest_date\"),\n        pl.col(\"date\").max().alias(\"latest_date\"),\n        pl.col(\"date\").n_unique().alias(\"unique_dates\")\n    ])\n    .collect()\n)\n\nearliest_date = date_info.item(0, \"earliest_date\")\nlatest_date = date_info.item(0, \"latest_date\")\nunique_dates = date_info.item(0, \"unique_dates\")\n\nprint(f\"Date Range: {earliest_date} to {latest_date}\")\nprint(f\"Total Days: {(latest_date - earliest_date).days + 1} days\")\nprint(f\"Unique Dates: {unique_dates:,} days with transactions\")\nprint(f\"Coverage: {unique_dates / ((latest_date - earliest_date).days +1 )* 100:.1f}% of total days\")\n\nmonthly_transactions = (\n    lazy_transactions\n    .with_columns([\n        pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\").dt.strftime(\"%Y-%m\").alias(\"year_month\")\n    ])\n    .group_by(\"year_month\")\n    .agg([\n        pl.len().alias(\"transaction_count\")\n    ])\n    .sort(\"year_month\")\n    .collect()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:38.805998Z","iopub.execute_input":"2025-09-23T20:20:38.806305Z","iopub.status.idle":"2025-09-23T20:20:54.121213Z","shell.execute_reply.started":"2025-09-23T20:20:38.806281Z","shell.execute_reply":"2025-09-23T20:20:54.120152Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* 100% daily coverage (734/734 days)\n* Rich Product Catalog with 105K articles with 25 detailed attributes. 99.6% complete / missing product descriptions.(department → product_type → color). \n* Zero missing values for Transactions (Total: 31M & 43K Transactions per day).\n* 1.37M Unique customers. Customer dataset missing values -->  FN: 65%, Active: 66%, 1.2% missing age data","metadata":{}},{"cell_type":"code","source":"print(\"COVERAGE\")\n\ntransaction_entities = (\n    lazy_transactions\n    .select([\n        pl.col(\"customer_id\").n_unique().alias(\"unique_customers_in_transactions\"),\n        pl.col(\"article_id\").n_unique().alias(\"unique_articles_in_transactions\")\n    ])\n    .collect()\n)\n\nunique_customers_in_trans = transaction_entities.item(0, \"unique_customers_in_transactions\")\nunique_articles_in_trans = transaction_entities.item(0, \"unique_articles_in_transactions\")\n\nprint(f\"Customers in dataset:     {customers.height:,}\")\nprint(f\"Customers with purchases: {unique_customers_in_trans:,}\")\nprint(f\"Articles in catalog:      {articles.height:,}\")\nprint(f\"Articles with sales:      {unique_articles_in_trans:,}\")\n\nprint(\"COVERAGE\")\n\ncustomer_coverage = (unique_customers_in_trans / customers.height) * 100\ncustomer_no_purchases = customers.height - unique_customers_in_trans\n\nprint(f\"Active customers:    {unique_customers_in_trans:,} ({customer_coverage:.1f}%)\")\nprint(f\"Inactive customers:  {customer_no_purchases:,} ({100-customer_coverage:.1f}%)\")\n\narticle_coverage = (unique_articles_in_trans / articles.height) * 100\narticles_no_sales = articles.height - unique_articles_in_trans\n\nprint(f\"Sold articles:       {unique_articles_in_trans:,} ({article_coverage:.1f}%)\")\nprint(f\"Never sold articles: {articles_no_sales:,} ({100-article_coverage:.1f}%)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:20:54.122216Z","iopub.execute_input":"2025-09-23T20:20:54.122548Z","iopub.status.idle":"2025-09-23T20:21:00.820053Z","shell.execute_reply.started":"2025-09-23T20:20:54.122511Z","shell.execute_reply":"2025-09-23T20:21:00.818699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"INTERACTION MATRIX:\")\n\ntotal_possible_interactions = unique_customers_in_trans * unique_articles_in_trans\nactual_interactions = transactions_count\nsparsity = (actual_interactions / total_possible_interactions) * 100\n\nprint(f\"Interaction Matrix Dimensions: {unique_customers_in_trans:,} × {unique_articles_in_trans:,}\")\nprint(f\"Possible interactions:         {total_possible_interactions:,}\")\nprint(f\"Actual interactions:           {actual_interactions:,}\")\nprint(f\"Matrix density:               {sparsity:.6f}%\")\nprint(f\"Sparsity:                     {100-sparsity:.4f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:21:00.821183Z","iopub.execute_input":"2025-09-23T20:21:00.821554Z","iopub.status.idle":"2025-09-23T20:21:00.828806Z","shell.execute_reply.started":"2025-09-23T20:21:00.821529Z","shell.execute_reply":"2025-09-23T20:21:00.827610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"| Metric | H&M Dataset | Industry Typical | Assessment |\n|--------|-------------|------------------|------------|\n| Customer Coverage | 99.3% | 85-95% | **Exceptional** |\n| Product Coverage | 99.1% | 80-90% | **Exceptional** |\n| Matrix Sparsity | 99.98% | 99.5-99.9% | **Standard** |\n| Interaction Volume | 32M | 10M-100M | **Substantial** |","metadata":{}},{"cell_type":"markdown","source":"### Customers","metadata":{}},{"cell_type":"markdown","source":"Our EDA for customers will answer these questions: \n\n* How do purchasing patterns differ across age groups?\n* How do customers evolve over time?\n* How does membership affect purchasing behavior and loyalty?\n* How consistent are customer preferences over time? (for collaborative filtering)\n* What are the customer-category relationship patterns? (for content-based filtering)\n* Are there customer clusters with similar behavior? (For neighborhood-based methods)","metadata":{}},{"cell_type":"markdown","source":"## 1 - How do purchasing patterns differ across age groups?","metadata":{}},{"cell_type":"code","source":"customer_purchase_behavior = (\n    lazy_transactions\n    .group_by('customer_id')\n    .agg([\n        pl.len().alias('total_purchases'),\n        pl.col('article_id').n_unique().alias('unique_articles'),\n        pl.col('price').sum().alias('total_spent'),\n        pl.col('price').mean().alias('avg_item_price'),\n        pl.col('t_dat').min().alias('first_purchase'),\n        pl.col('t_dat').max().alias('last_purchase')\n    ])\n    .with_columns([\n        pl.col('first_purchase').str.to_date(format='%Y-%m-%d'),\n        pl.col('last_purchase').str.to_date(format='%Y-%m-%d'),\n    ])\n    .with_columns([\n        (pl.col('last_purchase') - pl.col('first_purchase')).dt.total_days().alias('customer_lifespan_days'),\n        (pl.col('total_purchases') / ((pl.col('last_purchase') - pl.col('first_purchase')).dt.total_days() + 1)).alias('purchase_frequency'),\n        (pl.col('total_spent') / pl.col('total_purchases')).alias('avg_basket_value')\n    ])\n    .collect()\n)\n\ncustomer_demographics_behavior = (\n    customers\n    .join(customer_purchase_behavior, on='customer_id', how='inner')\n    .filter(pl.col('age').is_not_null())\n)\n\nprint(f\"Analyzing {len(customer_demographics_behavior):,} customers with complete age and purchase data\")\n\n# generations\ncustomer_segments = customer_demographics_behavior.with_columns([\n    pl.when(pl.col('age') < 25).then(pl.lit('Gen_Z'))\n    .when(pl.col('age') < 35).then(pl.lit('Millennial'))  \n    .when(pl.col('age') < 45).then(pl.lit('Gen_X_Young'))\n    .when(pl.col('age') < 55).then(pl.lit('Gen_X_Mature'))\n    .otherwise(pl.lit('Boomer_Plus'))\n    .alias('generation_segment'),\n    \n    # fashion age groups\n    pl.when(pl.col('age') < 20).then(pl.lit('Teen'))\n    .when(pl.col('age') < 25).then(pl.lit('Young_Adult'))\n    .when(pl.col('age') < 35).then(pl.lit('Professional'))\n    .when(pl.col('age') < 45).then(pl.lit('Established'))\n    .when(pl.col('age') < 55).then(pl.lit('Mature'))\n    .otherwise(pl.lit('Senior'))\n    .alias('fashion_age_segment'),\n    \n    # Customer value segments\n    pl.when(pl.col('total_spent') >= pl.col('total_spent').quantile(0.9))\n    .then(pl.lit('High_Value'))\n    .when(pl.col('total_spent') >= pl.col('total_spent').quantile(0.7))\n    .then(pl.lit('Medium_High'))\n    .when(pl.col('total_spent') >= pl.col('total_spent').quantile(0.3))\n    .then(pl.lit('Medium'))\n    .otherwise(pl.lit('Low_Value'))\n    .alias('value_segment')\n])\n\n# Age segment analysis\nage_segment_analysis = (\n    customer_segments\n    .group_by('generation_segment')\n    .agg([\n        pl.count().alias('customer_count'),\n        pl.col('age').mean().alias('avg_age'),\n        pl.col('total_purchases').mean().alias('avg_purchases'),\n        pl.col('unique_articles').mean().alias('avg_unique_items'),\n        pl.col('total_spent').mean().alias('avg_total_spent'),\n        pl.col('avg_item_price').mean().alias('avg_item_price'),\n        pl.col('purchase_frequency').mean().alias('avg_purchase_frequency'),\n        pl.col('customer_lifespan_days').mean().alias('avg_lifespan_days'),\n        \n        # Club membership penetration by age\n        (pl.col('club_member_status') == 'ACTIVE').mean().alias('club_membership_rate'),\n        \n        # Fashion news engagement\n        (pl.col('fashion_news_frequency') == 'Regularly').mean().alias('regular_news_rate')\n    ])\n    .sort('avg_age')\n)\n\nprint(\"GENERATIONAL CUSTOMER ANALYSIS:\")\nprint(\"Generation\".ljust(15) + \"Count\".ljust(10) + \"Avg Age\".ljust(10) + \"Avg Purchases\".ljust(15) + \n      \"Avg Spent\".ljust(12) + \"Club Rate\".ljust(12) + \"News Rate\")\nprint(\"-\" * 85)\n\nfor row in age_segment_analysis.iter_rows():\n    gen, count, age, purchases, spent, _, _, _, _, club_rate, news_rate = row\n    print(f\"{gen:<15} {count:<10,} {age:<10.1f} {purchases:<15.1f} ${spent:<11.0f} {club_rate:<11.1%} {news_rate:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:21:00.831130Z","iopub.execute_input":"2025-09-23T20:21:00.831453Z","iopub.status.idle":"2025-09-23T20:21:15.513382Z","shell.execute_reply.started":"2025-09-23T20:21:00.831429Z","shell.execute_reply":"2025-09-23T20:21:15.512409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_segments_pd = customer_segments.to_pandas()\n\nfig, axes = plt.subplots(2, 3, figsize=(20, 12))\nfig.suptitle('Customer Age Analysis', fontsize=16, fontweight='bold')\n\n# Age distribution with generations\nsns.histplot(data=customer_segments_pd, x='age', bins=50, kde=True, ax=axes[0,0])\naxes[0,0].axvline(customer_segments_pd['age'].mean(), color='red', linestyle='--', \n                  label=f'Mean: {customer_segments_pd[\"age\"].mean():.1f}')\naxes[0,0].set_title('Customer Age Distribution', fontsize=12)\naxes[0,0].legend()\n\n# Purchase behavior by generation\ngeneration_summary = customer_segments_pd.groupby('generation_segment').agg({\n    'total_purchases': 'mean',\n    'total_spent': 'mean',\n    'avg_item_price': 'mean'\n}).reset_index()\n\ngeneration_summary['customer_count'] = customer_segments_pd.groupby('generation_segment').size().values\n\nsns.barplot(data=customer_segments_pd, x='generation_segment', y='total_purchases', ax=axes[0,1])\naxes[0,1].set_title('Average Purchases by Generation', fontsize=12)\naxes[0,1].tick_params(axis='x', rotation=45)\n\n# Spending patterns by age\nsns.scatterplot(data=customer_segments_pd.sample(5000), x='age', y='total_spent', \n                hue='generation_segment', alpha=0.6, ax=axes[0,2])\naxes[0,2].set_title('Total Spending vs Age', fontsize=12)\naxes[0,2].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n\n# Club membership by age groups\nclub_by_age = customer_segments_pd.groupby('fashion_age_segment')['club_member_status'].apply(\n    lambda x: (x == 'ACTIVE').mean() * 100\n).reset_index()\nclub_by_age.columns = ['fashion_age_segment', 'membership_rate']\n\nsns.barplot(data=club_by_age, x='fashion_age_segment', y='membership_rate', ax=axes[1,0])\naxes[1,0].set_title('Club Membership Rate by Age Group', fontsize=12)\naxes[1,0].set_ylabel('Membership Rate (%)')\naxes[1,0].tick_params(axis='x', rotation=45)\n\n# Purchase diversity (unique articles) by age\nsns.boxplot(data=customer_segments_pd, x='generation_segment', y='unique_articles', ax=axes[1,1])\naxes[1,1].set_title('Purchase Diversity by Generation', fontsize=12)\naxes[1,1].tick_params(axis='x', rotation=45)\n\n# Fashion news engagement by age\nnews_by_age = customer_segments_pd.groupby('generation_segment')['fashion_news_frequency'].apply(\n    lambda x: (x == 'Regularly').mean() * 100\n).reset_index()\nnews_by_age.columns = ['generation_segment', 'regular_news_rate']\n\nsns.barplot(data=news_by_age, x='generation_segment', y='regular_news_rate', ax=axes[1,2])\naxes[1,2].set_title('Fashion News Engagement by Age', fontsize=12)\naxes[1,2].set_ylabel('Regular News Rate (%)')\naxes[1,2].tick_params(axis='x', rotation=45)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:21:15.514300Z","iopub.execute_input":"2025-09-23T20:21:15.514619Z","iopub.status.idle":"2025-09-23T20:21:50.808528Z","shell.execute_reply.started":"2025-09-23T20:21:15.514582Z","shell.execute_reply":"2025-09-23T20:21:50.807347Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now let's go back to our question: how do purchasing patterns differ across age groups? Now we have some answers based on our analysis. \n\n* H&M attracts broad range but youth-focused. Largest customer segment is young adults, the age distribution is right-skewed which means more younger customers than older.\n* There is a clear hiearchy between average purchases by generation : millenials > Gen X > Gen Z / Boomers. Millenial dominance with 29.6 avg purchase.\n* Both youngest and oldest buy less frequently. Peak spending in late 20s-30s.\n* Boomer suprise : highest news engagement despire lowest purchase.\n* Millenial paradox : lowest engagement rate despite highest spending.\n\nPurchasing patterns show 3 distinct behaviors for different age groups: \n* Gen Z --> High engagement, low spending --> digital first, trend-sensitive, budget-constrained.\n* Millenials --> Highest purchase, spending, and diversity --core customer base.\n* Older generations --> stable purchasing with high digital engagement.","metadata":{}},{"cell_type":"markdown","source":"### 2- How do customers evolve over time?","metadata":{}},{"cell_type":"code","source":"customer_evolution = (\n    lazy_transactions\n    .with_columns([\n        pl.col('t_dat').str.to_date(format='%Y-%m-%d').alias('purchase_date')\n    ])\n    .sort(['customer_id', 'purchase_date'])\n    .with_columns([\n        pl.int_range(pl.len()).over('customer_id').alias('purchase_order'),\n        pl.col('purchase_date').diff().over('customer_id').dt.total_days().alias('days_between')\n    ])\n    .group_by('customer_id')\n    .agg([\n        pl.col('purchase_date').min().alias('first_purchase'),\n        pl.col('purchase_date').max().alias('last_purchase'),\n        pl.col('purchase_date').count().alias('total_purchases'),\n        pl.col('price').sum().alias('lifetime_value'),\n        pl.col('price').mean().alias('avg_basket_value'),\n        pl.col('article_id').n_unique().alias('unique_items'),\n        pl.col('days_between').mean().alias('avg_interval'),\n        \n        # early vs recent behavior (first 3 vs last 3 purchases)\n        pl.col('price').head(3).mean().alias('early_spend'),\n        pl.col('price').tail(3).mean().alias('recent_spend'),\n        pl.col('article_id').head(5).n_unique().alias('early_diversity'),\n        pl.col('article_id').tail(5).n_unique().alias('recent_diversity')\n    ])\n    .with_columns([\n        (pl.col('last_purchase') - pl.col('first_purchase')).dt.total_days().alias('lifespan_days'),\n        (pl.col('recent_spend') / pl.col('early_spend')).alias('spend_growth'),\n        (pl.col('recent_diversity') / pl.col('early_diversity')).alias('diversity_growth')\n    ])\n    .with_columns([\n        pl.when(pl.col('lifespan_days') <= 30).then(pl.lit('New'))\n        .when(pl.col('lifespan_days') <= 180).then(pl.lit('Growing'))\n        .when(pl.col('lifespan_days') <= 365).then(pl.lit('Mature'))\n        .otherwise(pl.lit('Loyal'))\n        .alias('lifecycle_stage'),\n        \n        pl.when(pl.col('spend_growth') > 1.3).then(pl.lit('Increasing'))\n        .when(pl.col('spend_growth') < 0.8).then(pl.lit('Decreasing'))\n        .otherwise(pl.lit('Stable'))\n        .alias('spend_pattern')\n    ])\n    .filter(pl.col('total_purchases') >= 3)\n    .collect()\n)\n\ncustomer_evolution_demo = (\n    customer_evolution\n    .join(customers.select(['customer_id', 'age', 'club_member_status']), on='customer_id', how='left')\n    .filter(pl.col('age').is_not_null())\n)\n\nprint(f\"Analyzing evolution of {len(customer_evolution_demo):,} customers\")\n\nlifecycle_analysis = (\n    customer_evolution_demo\n    .group_by('lifecycle_stage')\n    .agg([\n        pl.count().alias('count'),\n        pl.col('total_purchases').mean().alias('avg_purchases'),\n        pl.col('lifetime_value').mean().alias('avg_ltv'),\n        pl.col('avg_basket_value').mean().alias('avg_basket'),\n        pl.col('unique_items').mean().alias('avg_diversity'),\n        pl.col('spend_growth').median().alias('median_spend_growth'),\n        (pl.col('spend_pattern') == 'Increasing').mean().alias('increasing_rate')\n    ])\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:21:50.809755Z","iopub.execute_input":"2025-09-23T20:21:50.810537Z","iopub.status.idle":"2025-09-23T20:22:32.400920Z","shell.execute_reply.started":"2025-09-23T20:21:50.810500Z","shell.execute_reply":"2025-09-23T20:22:32.399754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"LIFECYCLE STAGE:\")\n\nprint(\"Stage\".ljust(12) + \"Count\".ljust(10) + \"Avg Purch\".ljust(12) + \"Avg LTV\".ljust(10) + \"Spend Growth\".ljust(15) + \"Increasing %\")\nprint(\"-\" * 75)\nfor row in lifecycle_analysis.iter_rows():\n    stage, count, purchases, ltv, basket, diversity, growth, inc_rate = row\n    print(f\"{stage:<12} {count:<10,} {purchases:<12.1f} ${ltv:<9.0f} {growth:<15.2f} {inc_rate:.1%}\")\n\ndel customer_evolution, lifecycle_analysis\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:22:32.402101Z","iopub.execute_input":"2025-09-23T20:22:32.402457Z","iopub.status.idle":"2025-09-23T20:22:32.426869Z","shell.execute_reply.started":"2025-09-23T20:22:32.402426Z","shell.execute_reply":"2025-09-23T20:22:32.425645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Monthly cohort\nmonthly_cohorts = (\n    lazy_transactions\n    .with_columns([\n        pl.col('t_dat').str.to_date(format='%Y-%m-%d').alias('purchase_date')\n    ])\n    .group_by('customer_id')\n    .agg([\n        pl.col('purchase_date').min().alias('first_purchase'),\n        pl.col('purchase_date').count().alias('purchases')\n    ])\n    .with_columns([\n        pl.col('first_purchase').dt.strftime('%Y-%m').alias('cohort_month')\n    ])\n    .group_by('cohort_month')\n    .agg([\n        pl.count().alias('cohort_size'),\n        pl.col('purchases').mean().alias('avg_purchases')\n    ])\n    .sort('cohort_month')\n    .collect()\n)\n\n# Customer spending patterns\nspend_evolution = customer_evolution_demo.select([\n    'spend_pattern', 'lifecycle_stage', 'spend_growth', 'age'\n]).to_pandas()\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\nfig.suptitle('Customer Evolution Analysis Over Time', fontsize=16, fontweight='bold')\n\n# lifecycle distribution\nlifecycle_counts = customer_evolution_demo['lifecycle_stage'].value_counts()\nlifecycle_data = lifecycle_counts.to_pandas()\naxes[0,0].pie(lifecycle_data['count'], labels=lifecycle_data['lifecycle_stage'], autopct='%1.1f%%')\naxes[0,0].set_title('Customer Lifecycle Distribution')\n\n# Spending evolution by lifecycle\nsns.boxplot(data=spend_evolution, x='lifecycle_stage', y='spend_growth', ax=axes[0,1])\naxes[0,1].set_title('Spending Growth by Lifecycle Stage')\naxes[0,1].axhline(y=1, color='red', linestyle='--', alpha=0.7)\n\n# Age vs spending evolution\nsns.scatterplot(data=spend_evolution.sample(5000), x='age', y='spend_growth', \n                hue='lifecycle_stage', alpha=0.6, ax=axes[0,2])\naxes[0,2].set_title('Age vs Spending Evolution')\naxes[0,2].axhline(y=1, color='red', linestyle='--', alpha=0.7)\n\n# Spend patterns by lifecycle\nspend_pattern_cross = pd.crosstab(spend_evolution['lifecycle_stage'], \n                                  spend_evolution['spend_pattern'], normalize='index')\nspend_pattern_cross.plot(kind='bar', stacked=True, ax=axes[1,0])\naxes[1,0].set_title('Spending Patterns by Lifecycle Stage')\naxes[1,0].tick_params(axis='x', rotation=45)\n\n# Cohort size over time\ncohort_data = monthly_cohorts.to_pandas()\naxes[1,1].plot(cohort_data['cohort_month'], cohort_data['cohort_size'], marker='o')\naxes[1,1].set_title('Customer Acquisition by Cohort')\naxes[1,1].tick_params(axis='x', rotation=45)\naxes[1,1].set_ylabel('New Customers')\n\n# Customer value evolution\nlifecycle_summary = customer_evolution_demo.group_by('lifecycle_stage').agg([\n    pl.col('lifetime_value').mean().alias('avg_ltv')\n]).to_pandas()\naxes[1,2].bar(lifecycle_summary['lifecycle_stage'], lifecycle_summary['avg_ltv'])\naxes[1,2].set_title('Average LTV by Lifecycle Stage')\naxes[1,2].tick_params(axis='x', rotation=45)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:22:32.427902Z","iopub.execute_input":"2025-09-23T20:22:32.428234Z","iopub.status.idle":"2025-09-23T20:22:48.140083Z","shell.execute_reply.started":"2025-09-23T20:22:32.428211Z","shell.execute_reply":"2025-09-23T20:22:48.139095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cohort_data = monthly_cohorts.to_pandas()\n\nlifecycle_summary = (\n    customer_evolution_demo.group_by('lifecycle_stage')\n    .agg([\n        pl.col('lifetime_value').mean().alias('avg_ltv')\n    ])\n).to_pandas()\n\nevolution_insights = customer_evolution_demo.select([\n    pl.col('spend_growth').mean().alias('avg_spend_evolution'),\n    pl.col('diversity_growth').mean().alias('avg_diversity_evolution'),\n    (pl.col('spend_pattern') == 'Increasing').mean().alias('customers_increasing_spend'),\n    (pl.col('spend_pattern') == 'Decreasing').mean().alias('customers_decreasing_spend'),\n    pl.col('lifespan_days').mean().alias('avg_customer_lifespan'),\n    pl.col('avg_interval').mean().alias('avg_purchase_interval')\n])\n\nprint(f\"\\nKEY METRICS:\")\ninsights = evolution_insights.to_pandas().iloc[0]\nprint(f\"Average spending evolution: {insights['avg_spend_evolution']:.2f}x\")\nprint(f\"Customers increasing spend: {insights['customers_increasing_spend']:.1%}\")\nprint(f\"Customers decreasing spend: {insights['customers_decreasing_spend']:.1%}\")\nprint(f\"Average customer lifespan: {insights['avg_customer_lifespan']:.0f} days\")\nprint(f\"Average purchase interval: {insights['avg_purchase_interval']:.0f} days\")\n\ndel insights, cohort_data, lifecycle_summary, evolution_insights\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:22:48.141121Z","iopub.execute_input":"2025-09-23T20:22:48.141367Z","iopub.status.idle":"2025-09-23T20:22:48.347157Z","shell.execute_reply.started":"2025-09-23T20:22:48.141349Z","shell.execute_reply":"2025-09-23T20:22:48.346242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now let's answer this question: how do customers evolve over time? \n\n* The customer portfolio includes both long-term buyers and newer entrants.\n* Loyal is the dominant stage (about 40% of all customers) followed by mature and growing. New is minority.\n* Median spend growth skews upward for Loyal and Growing, with Loyal showing the widest spread. On average, Spend Growth is 1.11x overall, suggesting most customers increase or sustain spend as they move along the lifecycle.\n* Spending patterns by lifecycle stage:  stable across stages, few decreasing trends. This demonstrates most customers maintain or grow spend rather than sharply drop off.\n* Customer acquisition by cohort: a sharp early peak followed by a long tail decline, indicating front-loaded growth and the importance of retention to sustain revenue.","metadata":{}},{"cell_type":"markdown","source":"### 3- How does membership affect purchasing behaviors?","metadata":{}},{"cell_type":"code","source":"print(\"Aggregating transaction data to get per-customer statistics...\")\ncustomer_agg_query = (\n    lazy_transactions\n    .group_by(\"customer_id\")\n    .agg([\n        pl.len().alias(\"n_transactions\"),\n        pl.col(\"article_id\").n_unique().alias(\"n_unique_articles\"),\n        pl.col(\"price\").sum().alias(\"total_spend\"),\n        pl.col(\"t_dat\").n_unique().alias(\"n_purchase_days\")\n    ])\n)\ncustomer_agg_stats = customer_agg_query.collect()\n\ncustomer_loyalty_data = (\n    customers\n    .join(customer_agg_stats, on=\"customer_id\", how=\"left\")\n    .with_columns(\n        pl.col(\"club_member_status\").fill_null(\"UNKNOWN\").alias(\"club_member_status\")\n    )\n    .drop_nulls(subset=['n_transactions']) # Drop only customers with no transactions\n)\n\n\ncustomer_loyalty_pd = customer_loyalty_data.to_pandas()\n\nfig, axes = plt.subplots(1, 3, figsize=(22, 7))\nfig.suptitle('Impact of Club Membership on Purchasing Behavior', fontsize=18, fontweight='bold')\n\n# Average Total Items Purchased\nsns.boxenplot(data=customer_loyalty_pd, x='club_member_status', y='n_transactions', ax=axes[0], palette='viridis')\naxes[0].set_title('Total Items Purchased per Customer', fontsize=14)\naxes[0].set_xlabel('Club Member Status')\naxes[0].set_ylabel('Number of Transactions (Log Scale)')\naxes[0].set_yscale('log')\n\n# Average Total Spend\nsns.boxenplot(data=customer_loyalty_pd, x='club_member_status', y='total_spend', ax=axes[1], palette='plasma')\naxes[1].set_title('Total Spend per Customer', fontsize=14)\naxes[1].set_xlabel('Club Member Status')\naxes[1].set_ylabel('Total Spend (Log Scale)')\naxes[1].set_yscale('log')\n\n# Average Number of Active Shopping Days\nsns.boxenplot(data=customer_loyalty_pd, x='club_member_status', y='n_purchase_days', ax=axes[2], palette='magma')\naxes[2].set_title('Distinct Shopping Days per Customer', fontsize=14)\naxes[2].set_xlabel('Club Member Status')\naxes[2].set_ylabel('Number of Days (Log Scale)')\naxes[2].set_yscale('log')\n\nplt.tight_layout(rect=[0, 0.03, 1, 0.95])\nplt.show()\n\n# --- Clean up memory ---\ndel customer_agg_stats, customer_loyalty_data, customer_loyalty_pd\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:22:48.350305Z","iopub.execute_input":"2025-09-23T20:22:48.350579Z","iopub.status.idle":"2025-09-23T20:23:10.025682Z","shell.execute_reply.started":"2025-09-23T20:22:48.350561Z","shell.execute_reply":"2025-09-23T20:23:10.024589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The club membership is a powerful feature to understand engagement and intent with the brand. The active segment demonstrates a higher spend and frequency. They can be considered as brand loyalists and the brand's most valuable cohort because they are actively participating the ecosystem and they're more likely attuned to the trends and responsive to marketing. \n\nUnknowns and pre-creates looks nearly identical low engagement profiles. They are likely driven by immediate need or impulse buys rather than brand loyalty. For this group, the recommender system's goal is **conversion efficiency**. Showing them globally popular, high-probability \"safe bets\" (bestsellers, core items) is a more effective strategy than trying to predict a nuanced, personal taste that they have not yet revealed.\n\nWe can consider this strategy in the further steps: \n\n*   **`ACTIVE`** member -> Prioritize **personalization and discovery** (ALS, Content-Based).\n*   **`UNKNOWN`** or **`PRE-CREATE`** member -> Prioritize **popularity and high-conversion** items.\n*   **`LEFT CLUB`** member -> Trigger a **re-engagement strategy**.","metadata":{}},{"cell_type":"markdown","source":"### 4- What are distinct customer value tiers and their characteristics?","metadata":{}},{"cell_type":"code","source":"print(\"CUSTOMER TRANSACTION METRICS\")\n\ncustomer_metrics = (\n    lazy_transactions\n    .with_columns([\n        pl.col('t_dat').str.strptime(pl.Date, format='%Y-%m-%d'),\n        pl.col('price').cast(pl.Float64)\n    ])\n    .group_by('customer_id')\n    .agg([\n        # transaction frequency metrics\n        pl.count().alias('total_transactions'),\n        pl.col('price').sum().alias('total_spent'),\n        pl.col('price').mean().alias('avg_transaction_value'),\n        pl.col('price').std().alias('std_transaction_value'),\n        pl.col('article_id').n_unique().alias('unique_items_purchased'),\n        \n        # temporal metrics\n        pl.col('t_dat').min().alias('first_purchase_date'),\n        pl.col('t_dat').max().alias('last_purchase_date'),\n        pl.col('t_dat').n_unique().alias('purchase_days'),\n        \n        # behavioral metrics\n        pl.col('sales_channel_id').mode().first().alias('preferred_channel'),\n        pl.col('article_id').count().alias('total_items')\n    ])\n    .with_columns([\n        (pl.col('last_purchase_date') - pl.col('first_purchase_date')).dt.total_days().alias('customer_lifetime_days'),\n        (pl.col('total_spent') / pl.col('total_transactions')).alias('avg_basket_size'),\n        (pl.col('total_transactions') / pl.col('purchase_days')).alias('purchase_frequency'),\n        (pl.col('unique_items_purchased') / pl.col('total_transactions')).alias('item_variety_ratio')\n    ])\n    .collect()\n)\n\nprint(f\"Calculated metrics for {len(customer_metrics):,} customers\")\n\ncustomer_metrics = customer_metrics.with_columns([\n    pl.col('std_transaction_value').fill_null(0),\n    pl.col('customer_lifetime_days').fill_null(0),\n    pl.col('purchase_frequency').fill_null(pl.col('total_transactions')),\n])\n\nprint(\"\\nCUSTOMER VALUE TIERS:\")\n\nspending_50 = customer_metrics.select(pl.col('total_spent')).to_series().quantile(0.5)\nspending_80 = customer_metrics.select(pl.col('total_spent')).to_series().quantile(0.8)\nspending_95 = customer_metrics.select(pl.col('total_spent')).to_series().quantile(0.95)\n\nfrequency_50 = customer_metrics.select(pl.col('total_transactions')).to_series().quantile(0.5)\nfrequency_80 = customer_metrics.select(pl.col('total_transactions')).to_series().quantile(0.8)\nfrequency_95 = customer_metrics.select(pl.col('total_transactions')).to_series().quantile(0.95)\n\nrecency_date = customer_metrics.select(pl.col('last_purchase_date')).to_series().max()\n\nprint(f\"Spending thresholds: 50%=${spending_50:.2f}, 80%=${spending_80:.2f}, 95%=${spending_95:.2f}\")\nprint(f\"Frequency thresholds: 50%={frequency_50:.0f}, 80%={frequency_80:.0f}, 95%={frequency_95:.0f} transactions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:10.027095Z","iopub.execute_input":"2025-09-23T20:23:10.028076Z","iopub.status.idle":"2025-09-23T20:23:40.817276Z","shell.execute_reply.started":"2025-09-23T20:23:10.028033Z","shell.execute_reply":"2025-09-23T20:23:40.815825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_metrics.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:40.818924Z","iopub.execute_input":"2025-09-23T20:23:40.819336Z","iopub.status.idle":"2025-09-23T20:23:40.829336Z","shell.execute_reply.started":"2025-09-23T20:23:40.819307Z","shell.execute_reply":"2025-09-23T20:23:40.828146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_full = (\n    customer_metrics\n    .join(customers, on='customer_id', how='left')\n    .with_columns([\n        pl.col('age').fill_null(pl.col('age').median()),\n        pl.col('club_member_status').fill_null('UNKNOWN'),\n        pl.col('fashion_news_frequency').fill_null('NONE'),\n                \n        # recency score (days since last purchase)\n        (pl.date(2020, 9, 22) - pl.col('last_purchase_date')).dt.total_days().alias('days_since_last_purchase'),\n        \n        # engagement scores\n        (pl.col('total_transactions') * pl.col('avg_transaction_value')).alias('engagement_score'),\n        (pl.col('unique_items_purchased') / pl.col('total_transactions')).alias('exploration_ratio'),\n        \n        # behavior indicators\n        pl.when(pl.col('preferred_channel') == 1).then(pl.lit(1)).otherwise(pl.lit(0)).alias('online_shopper'),\n        pl.when(pl.col('preferred_channel') == 2).then(pl.lit(1)).otherwise(pl.lit(0)).alias('store_shopper'),\n    ])\n)\n\nprint(f\"Customer dataset created with {len(customer_full):,} customers and {customer_full.width} features\")\n\n\nkey_metrics = ['total_spent', 'total_transactions', 'avg_transaction_value', 'customer_lifetime_days', 'unique_items_purchased']\n\ndf_analysis = customer_full.select(key_metrics+['days_since_last_purchase']).to_pandas()\n\ndesc_stats = df_analysis.describe()\n\ndesc_stats.round(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:40.831136Z","iopub.execute_input":"2025-09-23T20:23:40.831473Z","iopub.status.idle":"2025-09-23T20:23:42.234516Z","shell.execute_reply.started":"2025-09-23T20:23:40.831442Z","shell.execute_reply":"2025-09-23T20:23:42.232599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for metric in key_metrics:\n    metric_skewness = skew(df_analysis[metric])\n    print(f\"{metric}: {metric_skewness:.3f} {'(highly skewed)' if abs(metric_skewness) > 2 else '(moderately skewed)' if abs(metric_skewness) > 0.5 else '(normal)'}\")\n\n\ncorrelation_matrix = df_analysis[key_metrics].corr()\nprint(\"Correlation Matrix:\")\ncorrelation_matrix.round(3)\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\nfig.suptitle('Customer Behavior Metrics Distribution Analysis', fontsize=16, fontweight='bold')\n\nfor i, metric in enumerate(key_metrics):\n    row, col = i // 3, i % 3\n    \n    data = df_analysis[metric]\n    if skew(data) > 2:\n        data = np.log1p(data)  # log(1+x) to handle zeros\n        title = f'Log({metric})'\n    else:\n        title = metric\n    \n    axes[row, col].hist(data, bins=50, alpha=0.7, edgecolor='black')\n    axes[row, col].set_title(title, fontweight='bold')\n    axes[row, col].set_xlabel('Value')\n    axes[row, col].set_ylabel('Frequency')\n    axes[row, col].grid(True, alpha=0.3)\n\n# Plot correlation heatmap in the last subplot\naxes[1, 2].clear()\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0, ax=axes[1, 2])\naxes[1, 2].set_title('Correlation Matrix', fontweight='bold')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:45.284908Z","iopub.execute_input":"2025-09-23T20:23:45.285207Z","iopub.status.idle":"2025-09-23T20:23:48.157222Z","shell.execute_reply.started":"2025-09-23T20:23:45.285184Z","shell.execute_reply":"2025-09-23T20:23:48.155940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix.round(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:48.158675Z","iopub.execute_input":"2025-09-23T20:23:48.159036Z","iopub.status.idle":"2025-09-23T20:23:48.172949Z","shell.execute_reply.started":"2025-09-23T20:23:48.158998Z","shell.execute_reply":"2025-09-23T20:23:48.171846Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Highly skewed customer behavior (top 20% drive 80% of value)\n* High spenders are also frequent shoppers (r=0.959)\n* Frequent customers buy more variety (r=0.985)\n* Customer lifetime strongly correlates with transaction frequency (r=0.567) --> loyalty impact\n* average order value is independent of other metrics -too broad options.\n","metadata":{}},{"cell_type":"markdown","source":"### Product Catalog Understanding","metadata":{}},{"cell_type":"code","source":"current_categories = articles['product_group_name'].value_counts().head(10)\nprint(\"Top product_group_name categories:\")\nprint(current_categories.to_pandas())\n\ndetailed_categories = articles['product_type_name'].value_counts().head(20)\nprint(\"\\nTop product_type_name categories:\")  \nprint(detailed_categories.to_pandas())\n\nprint(\"\\nProduct types within 'Garment Upper body':\")\nupper_body_types = (\n    articles\n    .filter(pl.col('product_group_name') == 'Garment Upper body')\n    ['product_type_name'].value_counts().head(10)\n)\nprint(upper_body_types.to_pandas())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:48.174142Z","iopub.execute_input":"2025-09-23T20:23:48.174496Z","iopub.status.idle":"2025-09-23T20:23:48.250015Z","shell.execute_reply.started":"2025-09-23T20:23:48.174472Z","shell.execute_reply":"2025-09-23T20:23:48.248397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hierarchy_cols = [col for col in articles.columns if any(keyword in col.lower() \n                  for keyword in ['department', 'category', 'group', 'type', 'section'])]\n\nhierarchy_analysis = {}\nfor col in hierarchy_cols:\n    unique_count = articles.select(pl.col(col).n_unique()).item()\n    hierarchy_analysis[col] = unique_count\n    print(f\"{col}: {unique_count} unique values\")\n\nmain_hierarchy = articles.select([\n    'product_type_name',\n    'product_group_name', \n    'graphical_appearance_name',\n    'colour_group_name',\n    'department_name',\n    'index_name',\n    'index_group_name',\n    'section_name',\n    'garment_group_name'\n]).to_pandas()\n\nprint(\"TOP PRODUCT CATEGORIES:\")\nfor col in ['department_name', 'product_group_name', 'product_type_name']:\n    print(f\"\\n{col.upper()}:\")\n    if col in main_hierarchy.columns:\n        top_categories = main_hierarchy[col].value_counts().head(10)\n        for category, count in top_categories.items():\n            print(f\"  {category}: {count:,} products ({count/len(main_hierarchy)*100:.1f}%)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:48.251792Z","iopub.execute_input":"2025-09-23T20:23:48.252201Z","iopub.status.idle":"2025-09-23T20:23:48.371564Z","shell.execute_reply.started":"2025-09-23T20:23:48.252170Z","shell.execute_reply":"2025-09-23T20:23:48.370421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hierarchy_levels = ['index_group_name', 'garment_group_name', 'product_group_name', 'product_type_name']\n\nhierarchy_data = (\n    articles\n    .group_by(['index_group_name', 'garment_group_name', 'product_group_name', 'product_type_name'])\n    .agg(pl.count().alias('product_count'))\n    .sort('product_count', descending=True)\n    .to_pandas()\n)\n\nprint(f\"Total hierarchy combinations: {len(hierarchy_data)}\")\n\nfig_treemap_corrected = px.treemap(\n    hierarchy_data.head(100),\n    path=['index_group_name', 'garment_group_name', 'product_group_name', 'product_type_name'],\n    values='product_count',\n    title='H&M Product Hierarchy - Corrected Treemap (Index Group → Garment Group → Product Group → Product Type)',\n    color='product_count',\n    color_continuous_scale='Viridis',\n    height=700\n)\n\nfig_treemap_corrected.update_traces(textinfo=\"label+value\")\nfig_treemap_corrected.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:48.372743Z","iopub.execute_input":"2025-09-23T20:23:48.373145Z","iopub.status.idle":"2025-09-23T20:23:51.803152Z","shell.execute_reply.started":"2025-09-23T20:23:48.373110Z","shell.execute_reply":"2025-09-23T20:23:51.801954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles_pd = articles.to_pandas()\n\n# Top 15 Product Groups\nplt.figure(figsize=(12, 8))\nsns.countplot(y='product_group_name', data=articles_pd, order=articles_pd['product_group_name'].value_counts().head(15).index, palette='mako')\nplt.title('Top 15 Most Common Product Groups in Catalog', fontsize=16)\nplt.xlabel('Number of Articles')\nplt.ylabel('Product Group')\nplt.show()\n\n# --- Clean up memory ---\ndel articles_pd\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:51.804306Z","iopub.execute_input":"2025-09-23T20:23:51.804759Z","iopub.status.idle":"2025-09-23T20:23:52.547036Z","shell.execute_reply.started":"2025-09-23T20:23:51.804716Z","shell.execute_reply":"2025-09-23T20:23:52.546028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:52.548174Z","iopub.execute_input":"2025-09-23T20:23:52.548539Z","iopub.status.idle":"2025-09-23T20:23:52.558007Z","shell.execute_reply.started":"2025-09-23T20:23:52.548515Z","shell.execute_reply":"2025-09-23T20:23:52.556691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## COLOR & MATERIAL // ARTICLES\n\ncolor_columns = [col for col in articles.columns if 'colour' in col.lower()]\n\nprint('color related columns in the articles dataset:\\n')\nprint(color_columns)\n\nfor col in color_columns: \n    unique_colors = articles.select(pl.col(col).n_unique()).item()\n    print(f\"Column: {col} // Unique Color Count: {unique_colors}\")\n\n# color distribution\n\ncolor_distribution = (\n    articles\n    .group_by('colour_group_name')\n    .agg([\n        pl.count().alias('product_count'),\n        pl.col('product_group_name').n_unique().alias('product_groups_count'),\n        pl.col('department_name').n_unique().alias('departments_count')\n    ])\n    .sort('product_count', descending=True)\n    .to_pandas()\n)\n\nprint(f\"\\nTotal Unique Colors: {len(color_distribution)}\")\nprint(\"\\nTOP 15 COLORS BY PRODUCT COUNT:\")\ntop_colors = color_distribution.head(15)\nfor _, row in top_colors.iterrows():\n    color = row['colour_group_name']\n    count = row['product_count']\n    pct = count / len(articles) * 100\n    groups = row['product_groups_count']\n    depts = row['departments_count']\n    print(f\"  {color}: {count:,} products ({pct:.1f}%) | {groups} groups | {depts} departments\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:23:52.559112Z","iopub.execute_input":"2025-09-23T20:23:52.559401Z","iopub.status.idle":"2025-09-23T20:23:52.617660Z","shell.execute_reply.started":"2025-09-23T20:23:52.559372Z","shell.execute_reply":"2025-09-23T20:23:52.616665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T19:19:08.377140Z","iopub.execute_input":"2025-09-23T19:19:08.377571Z","iopub.status.idle":"2025-09-23T19:19:08.387658Z","shell.execute_reply.started":"2025-09-23T19:19:08.377540Z","shell.execute_reply":"2025-09-23T19:19:08.386586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_list = ['product_group_name','garment_group_name', 'index_group_name', 'index_name', 'section_name',\n                'colour_group_name', 'perceived_colour_value_name', 'graphical_apperance_name'\n               ]\n\nfor feature in feature_list:\n    if feature in articles.columns:\n        unique_count = articles.select(pl.col(feature).n_unique()).item()\n        print(f\"{feature} : {unique_count} unique values\")\n\n\n# color-category combinations\n\ncolor_category_matrix = (\n    articles\n    .group_by(['index_name', 'perceived_colour_value_name'])\n    .agg(pl.count().alias('combination_count'))\n    .filter(pl.col('combination_count')>10)\n    .sort('combination_count', descending=True)\n    .head(25)\n    .to_pandas()\n)\n\ncolor_category_matrix","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:40:35.691424Z","iopub.execute_input":"2025-09-23T20:40:35.691821Z","iopub.status.idle":"2025-09-23T20:40:35.740561Z","shell.execute_reply.started":"2025-09-23T20:40:35.691792Z","shell.execute_reply":"2025-09-23T20:40:35.739442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"attribute_idf = {}\n\nproduct_group = articles.select('section_name').to_pandas()['section_name'].value_counts()\n\nfor group, count in product_group.head(10).items():\n    frequency_count = count/len(articles)\n    idf_score = np.log(len(articles)/count) # measure how rare a feature is across a collection.\n    attribute_idf[group] = idf_score\n\nattribute_idf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:54:51.920844Z","iopub.execute_input":"2025-09-23T20:54:51.921236Z","iopub.status.idle":"2025-09-23T20:54:51.944054Z","shell.execute_reply.started":"2025-09-23T20:54:51.921209Z","shell.execute_reply":"2025-09-23T20:54:51.943233Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### What is IDF? \n\n* IDF measures how rare a term/feature is across a collection of documents/items.\n* IDF = log(Total Documents / Documents Containing Term)\n* It's a very useful metric for recommendation systems but it would penalize in popular or trending items.","metadata":{}},{"cell_type":"code","source":"\n# perceived_colour_value_name for material hints\nif 'perceived_colour_value_name' in articles.columns:\n    perceived_colors = articles.select('perceived_colour_value_name').to_pandas()['perceived_colour_value_name'].value_counts()\n    print(\"TOP PERCEIVED COLOR VALUES (may contain material info):\")\n    print(perceived_colors.head(20))\n\nif 'detail_desc' in articles.columns:\n    print(f\"\\nDETAIL DESCRIPTIONS:\")\n    detail_desc_sample = articles.select('detail_desc').filter(pl.col('detail_desc').is_not_null()).head(10).to_pandas()\n    print(\"Sample detail descriptions:\")\n    for desc in detail_desc_sample['detail_desc']:\n        print(f\"  - {desc}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T20:25:46.080885Z","iopub.execute_input":"2025-09-23T20:25:46.081264Z","iopub.status.idle":"2025-09-23T20:25:46.107797Z","shell.execute_reply.started":"2025-09-23T20:25:46.081236Z","shell.execute_reply":"2025-09-23T20:25:46.106786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weekly_sales_query = (\n    lazy_transactions\n    .with_columns(\n        pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\") \n    )\n    .with_columns(\n        pl.col(\"t_dat\").dt.truncate(\"1w\").alias(\"week\") \n    )\n    .group_by(\"week\")\n    .agg(\n        pl.count().alias(\"n_transactions\")\n    )\n    .sort(\"week\")\n)\n\nprint(\"Calculating weekly sales for the entire dataset...\")\nweekly_sales = weekly_sales_query.collect()\n\nweekly_sales_pd = weekly_sales.to_pandas()\n\nfig = px.line(weekly_sales_pd, x='week', y='n_transactions', title='Weekly Sales Transactions (Full Dataset)')\nfig.show()\n\ndel weekly_sales_pd, weekly_sales\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T16:43:08.836435Z","iopub.execute_input":"2025-09-23T16:43:08.836807Z","iopub.status.idle":"2025-09-23T16:43:14.539440Z","shell.execute_reply.started":"2025-09-23T16:43:08.836785Z","shell.execute_reply":"2025-09-23T16:43:14.538762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"article_first_sale_query = (\n    lazy_transactions\n    .with_columns(pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\"))\n    .group_by(\"article_id\")\n    .agg(\n        pl.col(\"t_dat\").min().alias(\"launch_date\")\n    )\n)\narticle_first_sale = article_first_sale_query.collect()\n\nmonthly_sales_query = (\n    lazy_transactions\n    .with_columns(pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\"))\n    .join(articles.lazy(), on='article_id', how='left') \n    .with_columns(\n        pl.col(\"t_dat\").dt.month().alias(\"month\") \n    )\n    .group_by([\"month\", \"product_group_name\"])\n    .agg([\n        pl.len().alias(\"n_transactions\"),\n        pl.col(\"article_id\").n_unique().alias(\"n_unique_articles_in_month\") \n    ])\n)\nmonthly_sales = monthly_sales_query.collect()\nprint(\"Monthly sales calculation complete.\")\n\n\n# We are now normalizing by the number of unique articles sold in that period.\n# This gives us a \"sales per active product\", which handles product maturity.\n# A new product can only contribute to sales in the months it's active.\nmonthly_sales = monthly_sales.with_columns(\n    (pl.col(\"n_transactions\") / pl.col(\"n_unique_articles_in_month\")).alias(\"sales_per_active_article\")\n)\n\nmonthly_sales_pd = monthly_sales.to_pandas()\n\ntop_12_groups = articles['product_group_name'].value_counts().get_column('product_group_name').to_list()\nmonthly_sales_top_pd = monthly_sales_pd[monthly_sales_pd['product_group_name'].isin(top_12_groups)]\n\npivot_table = monthly_sales_top_pd.pivot_table(\n    index='product_group_name', \n    columns='month', \n    values='sales_per_active_article',\n    fill_value=0\n)\n\npivot_table_normalized = pivot_table.div(pivot_table.sum(axis=1), axis=0)\n\nplt.figure(figsize=(16, 10))\nsns.heatmap(pivot_table_normalized, cmap='YlGnBu', annot=False)\nplt.title('Seasonal Demand: Normalized Sales Intensity per Product Group', fontsize=18)\nplt.xlabel('Month of the Year')\nplt.ylabel('Product Group')\nplt.show()\n\n# --- Clean up memory ---\ndel article_first_sale, monthly_sales, monthly_sales_pd, monthly_sales_top_pd, pivot_table, pivot_table_normalized\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T16:44:05.113317Z","iopub.execute_input":"2025-09-23T16:44:05.113701Z","iopub.status.idle":"2025-09-23T16:44:16.680821Z","shell.execute_reply.started":"2025-09-23T16:44:05.113675Z","shell.execute_reply":"2025-09-23T16:44:16.679994Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* The seasonal patterns are much clearer now. The heatmap shows a massive spike in September for 'Fun' category. This category likely corresponds to a specific seasonal event, perhaps 'back to school' or a particular holiday promotion.\n* Similarly, the 'stationery' shows a strong peak in May, which might be related to 'end-of-school' year preparations or a specific marketing campaign.\n* Other categories show more clear trends.Swimwear, shorts, and garment show higher intensity in the warmer months, while heavier items like 'socks & tights' show intensity in the colder months.\n* Categories like 'underwear' and 'garment upper body' show relatively consistent across all months. This indicates they are products with stable, year-round demand. In contrast, categories like 'fun' and 'stationery' are more seasonal. Their demand is concentrated in specific periods.\n* We will use this knowledge in the further steps because time of the year and 'season' would be our one of the most powerful predictors of what a customer might be interested in. ","metadata":{}},{"cell_type":"code","source":"# Long Tail Analysis (Maturity-aware) ---\n\nprint(\"Calculating article launch dates and total sales from full dataset...\")\narticle_sales_and_launch_query = (\n    lazy_transactions\n    .with_columns(pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\"))\n    .group_by(\"article_id\")\n    .agg([\n        pl.len().alias(\"n_sales\"),\n        pl.col(\"t_dat\").min().alias(\"launch_date\")\n    ])\n)\n\narticle_stats = article_sales_and_launch_query.collect()\n\ndataset_end_date = article_stats.select(pl.col(\"launch_date\").max()).item()\narticle_stats = article_stats.with_columns(\n    ((pl.lit(dataset_end_date) - pl.col(\"launch_date\")).dt.total_days() / 7 + 1)\n    .cast(pl.Int32)\n    .alias(\"article_age_weeks\")\n)\n\n\narticle_stats = article_stats.with_columns(\n    (pl.col(\"n_sales\") / pl.col(\"article_age_weeks\")).alias(\"sales_per_week\")\n)\n\narticle_stats_sorted = article_stats.sort('n_sales', descending=True)\n\nsales_numpy = article_stats_sorted['n_sales'].to_numpy()\n\ncumulative_sales_perc_numpy = 100 * sales_numpy.cumsum() / sales_numpy.sum()\ncumulative_articles_perc_numpy = 100 * (np.arange(1, len(sales_numpy) + 1)) / len(sales_numpy)\n\narticle_stats_final = article_stats_sorted.with_columns([\n    pl.Series(\"cumulative_sales_perc\", cumulative_sales_perc_numpy),\n    pl.Series(\"cumulative_articles_perc\", cumulative_articles_perc_numpy)\n])\n\n\narticle_sales_pd = article_stats_final.to_pandas()\n\nfig = px.line(\n    article_sales_pd, \n    x='cumulative_articles_perc', \n    y='cumulative_sales_perc',\n    title='The Long Tail: Sales Concentration (Full Dataset)',\n    labels={'cumulative_articles_perc': '% of Top Articles (Ranked by Raw Sales)', 'cumulative_sales_perc': '% of Total Sales'}\n)\nfig.add_hline(y=80, line_dash=\"dash\", annotation_text=\"80% of Sales\", annotation_position=\"bottom right\")\nfig.add_vline(x=20, line_dash=\"dash\", annotation_text=\"20% of Articles\")\nfig.show()\n\n# --- Final clean up ---\ndel article_stats, article_stats_sorted, article_stats_final, article_sales_pd\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:17:40.176941Z","iopub.execute_input":"2025-09-23T21:17:40.177400Z","iopub.status.idle":"2025-09-23T21:17:47.048288Z","shell.execute_reply.started":"2025-09-23T21:17:40.177361Z","shell.execute_reply":"2025-09-23T21:17:47.046817Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The plot above, analyzes the sales concentration across the entire two-year dataset, and it shows 'where the revenue comes from' for H&M business. \n* The curve rises very quickly at the beginning, and shows us approx 20% of the top-selling articles are responsible for 80% of all sales transactions.\n* The long part of the curve means that the remaining 80% of articles in h&m's catalog are responsible for only 20% of sales.\n* This pattern is very normal because fashion retail is often built on a foundation of core items. A basic black t-shirt or a pair blue jeans is purchased far more frequently than a niche, high-fashion item. Our recommendation system *must* be aware of this.\n\nNow we need some additional answers to navigate our strategy: \n\n1. Do newly released products sell more, or do classic products that have been on the market for a long time?\n2. How long does a product's popularity last?\n3. Do the types of products customers buy change over time?\n4. When is a customer likely to make their next purchase after making a purchase?","metadata":{}},{"cell_type":"code","source":"# Customer Purchase Frequency\ncustomer_purchases_query = (\n    lazy_transactions\n    .group_by('customer_id')\n    .agg(\n        pl.len().alias('n_purchases')\n    )\n)\n\nprint(\"\\nCalculating purchase frequency for all customers...\")\ncustomer_purchases = customer_purchases_query.collect()\nprint(\"Calculation complete.\")\n\n# Convert to pandas for plotting\ncustomer_purchases_pd = customer_purchases.to_pandas()\n\nplt.figure(figsize=(12, 6))\nsns.histplot(customer_purchases_pd['n_purchases'], bins=100, log_scale=(False, True))\nplt.title('Distribution of Number of Purchases per Customer (Full Dataset, Log Scale)')\nplt.xlabel('Total Number of Articles Purchased')\nplt.ylabel('Number of Customers (Log Scale)')\nplt.show()\n\nprint(customer_purchases_pd['n_purchases'].describe())\n\n# --- Clean up memory ---\ndel customer_purchases_pd, customer_purchases\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:18:15.860640Z","iopub.execute_input":"2025-09-23T21:18:15.861016Z","iopub.status.idle":"2025-09-23T21:18:42.358415Z","shell.execute_reply.started":"2025-09-23T21:18:15.860990Z","shell.execute_reply":"2025-09-23T21:18:42.357484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Calculating article first appearance dates and total sales...\")\narticle_stats_query = (\n    lazy_transactions\n    .with_columns(pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\"))\n    .group_by(\"article_id\")\n    .agg([\n        pl.len().alias(\"n_sales\"),\n        pl.col(\"t_dat\").min().alias(\"first_seen_date\")\n    ])\n)\narticle_stats = article_stats_query.collect()\nprint(\"Calculation complete.\")\n\n# how long the product has been available in the observed transaction window.\ndataset_end_date = article_stats.select(pl.col(\"first_seen_date\").max()).item()\narticle_stats = article_stats.with_columns(\n    ((pl.lit(dataset_end_date) - pl.col(\"first_seen_date\")).dt.total_days() / 7 + 1)\n    .cast(pl.Int32)\n    .alias(\"article_age_weeks_in_data\")\n)\nprint(\"Article age calculation complete.\")\n\n\n# sales velocity (sales per week since first seen)\narticle_stats = article_stats.with_columns(\n    (pl.col(\"n_sales\") / pl.col(\"article_age_weeks_in_data\")).alias(\"sales_per_week\")\n)\n\n\n# sales velocity based on article age\nage_labels = [\"New (0-4 wks)\", \"1-3 Months\", \"3-6 Months\", \"6-12 Months\", \"1-2 Years\", \">2 Years\"]\narticle_stats = article_stats.with_columns(\n    pl.when(pl.col(\"article_age_weeks_in_data\") <= 4).then(pl.lit(age_labels[0]))\n    .when(pl.col(\"article_age_weeks_in_data\") <= 13).then(pl.lit(age_labels[1]))\n    .when(pl.col(\"article_age_weeks_in_data\") <= 26).then(pl.lit(age_labels[2]))\n    .when(pl.col(\"article_age_weeks_in_data\") <= 52).then(pl.lit(age_labels[3]))\n    .when(pl.col(\"article_age_weeks_in_data\") <= 104).then(pl.lit(age_labels[4]))\n    .otherwise(pl.lit(age_labels[5]))\n    .alias(\"age_bin\")\n)\n\nage_bin_velocity = (\n    article_stats\n    .group_by(\"age_bin\")\n    .agg(pl.col(\"sales_per_week\").mean().alias(\"avg_sales_velocity\"))\n)\n\nage_bin_velocity_pd = age_bin_velocity.to_pandas()\nage_bin_velocity_pd['age_bin'] = pd.Categorical(age_bin_velocity_pd['age_bin'], categories=age_labels, ordered=True)\nage_bin_velocity_pd = age_bin_velocity_pd.sort_values('age_bin')\n\nplt.figure(figsize=(12, 7))\nsns.barplot(data=age_bin_velocity_pd, x=\"age_bin\", y=\"avg_sales_velocity\", palette=\"magma\")\nplt.title('Average Sales Velocity by Article Age (Time Since First Seen in Data)', fontsize=16)\nplt.xlabel('Article Age Since First Appearance')\nplt.ylabel('Average Sales per Week')\nplt.show()\n\n# --- Clean up memory ---\ndel article_stats, age_bin_velocity, age_bin_velocity_pd\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T16:46:37.901462Z","iopub.execute_input":"2025-09-23T16:46:37.902092Z","iopub.status.idle":"2025-09-23T16:46:42.397537Z","shell.execute_reply.started":"2025-09-23T16:46:37.902063Z","shell.execute_reply":"2025-09-23T16:46:42.396574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Sales velocity is the highest in the first three months after a product is introduced (appeared). This is likely driven by initial marketing pushes, seasonal relevance, and customer interest. After 3 months, there is a clear decay in the popularity which sounds reasonable regarding seasons of fashion industry. \n\n* We must derive a feature representing the age of a product. Our reranking model needs to learn this powerful pattern: newer items are, on average, much more likely to be purchased. \n* Our candidate generation must be dynamic and have a strong component that prioritizes 'trending' or 'newly popular' items.\n* This analysis also demonstrates the importance of 'cold start' problem. Since the highest sales velocity might occurs when an item is new and has little-to-no interaction data, the only way to effectively recommend it is by understanding its content (category, color, description) and matching it to a user's taste. ","metadata":{}},{"cell_type":"code","source":"print(\"HALF-LIFE ANALYSIS\")\n\nweekly_article_sales = (\n    lazy_transactions\n    .with_columns(pl.col(\"t_dat\").str.to_date(format=\"%Y-%m-%d\"))\n    .with_columns(pl.col(\"t_dat\").dt.truncate(\"1w\").alias(\"week\"))\n    .group_by([\"article_id\", \"week\"])\n    .agg(pl.len().alias(\"n_sales\"))\n    .sort([\"article_id\", \"week\"])\n    .collect()\n)\nprint(f\"Weekly sales calculated for {weekly_article_sales['article_id'].n_unique()} unique articles.\")\n\n\narticle_stats = (\n    weekly_article_sales\n    .group_by('article_id')\n    .agg(\n        pl.sum('n_sales').alias('total_sales'),\n        pl.max('n_sales').alias('peak_sales'),\n        pl.count('week').alias('weeks_active'),\n        pl.min('week').alias('first_week'),\n        pl.max('week').alias('last_week'),\n    )\n    .filter(\n        (pl.col('weeks_active') >= 3) &\n        (pl.col('total_sales') >= 10) &\n        (((pl.col('last_week') - pl.col('first_week')).dt.total_days() / 7) >= 2)\n    )\n)\n\npeak_sales_info = (\n    weekly_article_sales\n    .join(article_stats, on='article_id', how='inner')\n    # where sales match the article's peak sales\n    .filter(pl.col('n_sales') == pl.col('peak_sales'))\n    .group_by('article_id')\n    # if multiple weeks have peak sales, take the earliest one\n    .agg(pl.min('week').alias('peak_week'))\n    # join the peak_week back with the rest of the stats\n    .join(article_stats, on='article_id', how='inner')\n)\n\nprint(f\"Valid articles and peak week identified for {len(peak_sales_info)} articles.\")\n\nhalf_life_calculation_base = (\n    weekly_article_sales\n    .join(peak_sales_info.select(['article_id', 'peak_week', 'total_sales']), on='article_id', how='inner')\n)\n\n# Find the week where 50% of post-peak sales are consumed\nhalf_life_data = (\n    half_life_calculation_base\n    .sort(['article_id', 'week'])\n    .to_pandas()\n    .assign(\n        cumulative_sales=lambda df: df.groupby('article_id')['n_sales'].cumsum()\n    )\n    .pipe(pl.from_pandas) \n    .with_columns([\n        # cumulative sales AT peak week\n        pl.when(pl.col('week') == pl.col('peak_week'))\n        .then(pl.col('cumulative_sales'))\n        .otherwise(None)\n        .max().over('article_id').alias('sales_up_to_peak')\n    ])\n    .with_columns([\n        (pl.col('total_sales') - pl.col('sales_up_to_peak')).alias('total_remaining_from_peak')\n    ])\n    .filter(pl.col('week') > pl.col('peak_week'))\n    .with_columns([\n        (pl.col('total_sales') - pl.col('cumulative_sales')).alias('remaining_sales')\n    ])\n    .with_columns([\n        pl.when(pl.col('total_remaining_from_peak') > 0)\n        .then(1 - (pl.col('remaining_sales') / pl.col('total_remaining_from_peak')))\n        .otherwise(1.0)\n        .alias('remaining_fraction_consumed')\n    ])\n    .filter(pl.col('remaining_fraction_consumed') >= 0.5)\n    .group_by('article_id')\n    .agg([\n        pl.col('week').min().alias('half_life_week'),\n        pl.col('remaining_fraction_consumed').min().alias('actual_fraction')\n    ])\n)\n\n\nprint(f\"Half-life calculated for {len(half_life_data)} articles.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:19:21.521705Z","iopub.execute_input":"2025-09-23T21:19:21.522092Z","iopub.status.idle":"2025-09-23T21:19:31.106498Z","shell.execute_reply.started":"2025-09-23T21:19:21.522066Z","shell.execute_reply":"2025-09-23T21:19:31.105518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_stats = (\n    peak_sales_info\n    .join(half_life_data, on='article_id', how='inner')\n    .with_columns(\n        (((pl.col('half_life_week') - pl.col('peak_week')).dt.total_days() / 7)\n         .alias('half_life_weeks')),\n        (pl.col('peak_sales') / pl.col('total_sales')).alias('peak_intensity')\n    )\n    .filter(\n        (pl.col('half_life_weeks') >= 0) &\n        (pl.col('half_life_weeks') <= 52) # Cap at 1 year\n    )\n)\n\nprint(f\"Final dataset created for {len(final_stats)} articles.\")\n\nfinal_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:19:55.435818Z","iopub.execute_input":"2025-09-23T21:19:55.436893Z","iopub.status.idle":"2025-09-23T21:19:55.464403Z","shell.execute_reply.started":"2025-09-23T21:19:55.436828Z","shell.execute_reply":"2025-09-23T21:19:55.463353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_data_with_groups = (\n    final_stats\n    .join(\n        articles.select(['article_id', 'product_group_name']),\n        on='article_id',\n        how='left'\n    )\n    .filter(pl.col('product_group_name').is_not_null())\n)\n\ncategory_counts = final_data_with_groups['product_group_name'].value_counts()\nsignificant_categories = (\n    category_counts\n    .filter(pl.col('count') >= 50)\n    .get_column('product_group_name')\n    .to_list()\n)\n\nplot_data = final_data_with_groups.filter(\n    pl.col('product_group_name').is_in(significant_categories)\n)\n\nprint(f\"Dataset for visualization: {len(plot_data)} articles across {len(significant_categories)} categories.\")\n\nplot_order = (\n    plot_data\n    .group_by('product_group_name')\n    .agg(pl.col('half_life_weeks').median())\n    .sort('half_life_weeks', descending=True)\n    .get_column('product_group_name')\n    .to_list()\n)\n\nplot_data_pd = plot_data.to_pandas()\n\nfig, ax = plt.subplots(1, 1, figsize=(12, 10))\nfig.suptitle('Product Popularity Half-Life Analysis', fontsize=16, fontweight='bold')\n\nsns.boxenplot(\n    data=plot_data_pd,\n    x='half_life_weeks',\n    y='product_group_name',\n    order=plot_order,\n    palette='viridis',\n    ax=ax\n)\nax.set_title('Distribution of Popularity Half-Life by Product Group', fontsize=12)\nax.set_xlabel('Half-Life (Weeks after Peak Sales)')\nax.set_ylabel('Product Group')\nax.set_xlim(0, 30)\nax.grid(True, alpha=0.3)\n\nplt.tight_layout(rect=[0, 0, 1, 0.96])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:20:08.437122Z","iopub.execute_input":"2025-09-23T21:20:08.437453Z","iopub.status.idle":"2025-09-23T21:20:09.074038Z","shell.execute_reply.started":"2025-09-23T21:20:08.437429Z","shell.execute_reply":"2025-09-23T21:20:09.072286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"POPULARITY LIFECYCLE:\")\n\nlifecycle_summary = final_stats.select([\n    pl.col('half_life_weeks').mean().alias('avg_popularity_duration'),\n    pl.col('half_life_weeks').median().alias('median_popularity_duration'),\n    pl.col('peak_intensity').mean().alias('avg_peak_concentration'),\n    pl.col('weeks_active').mean().alias('avg_total_lifespan'),\n    (pl.col('half_life_weeks') / pl.col('weeks_active')).mean().alias('avg_decay_ratio')\n])\n\nlifecycle_summary.to_pandas().round(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:20:19.380876Z","iopub.execute_input":"2025-09-23T21:20:19.381268Z","iopub.status.idle":"2025-09-23T21:20:19.400428Z","shell.execute_reply.started":"2025-09-23T21:20:19.381240Z","shell.execute_reply":"2025-09-23T21:20:19.399233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Popularity patterns\npersistence_analysis = final_stats.select([\n    (pl.col('half_life_weeks') <= 2).sum().alias('very_short_≤2wks'),\n    ((pl.col('half_life_weeks') > 2) & (pl.col('half_life_weeks') <= 6)).sum().alias('short_3-6wks'),\n    ((pl.col('half_life_weeks') > 6) & (pl.col('half_life_weeks') <= 12)).sum().alias('medium_7-12wks'),\n    ((pl.col('half_life_weeks') > 12) & (pl.col('half_life_weeks') <= 20)).sum().alias('long_13-20wks'),\n    (pl.col('half_life_weeks') > 20).sum().alias('very_long_>20wks'),\n    pl.count().alias('total_products')\n]).with_columns([\n    (pl.col('very_short_≤2wks') / pl.col('total_products') * 100).alias('very_short_pct'),\n    (pl.col('short_3-6wks') / pl.col('total_products') * 100).alias('short_pct'),\n    (pl.col('medium_7-12wks') / pl.col('total_products') * 100).alias('medium_pct'),\n    (pl.col('long_13-20wks') / pl.col('total_products') * 100).alias('long_pct'),\n    (pl.col('very_long_>20wks') / pl.col('total_products') * 100).alias('very_long_pct')\n])\n\nprint(\"\\nPOPULARITY PERSISTENCE PATTERNS:\")\npersistence_analysis.to_pandas().round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:20:28.848994Z","iopub.execute_input":"2025-09-23T21:20:28.849389Z","iopub.status.idle":"2025-09-23T21:20:28.876377Z","shell.execute_reply.started":"2025-09-23T21:20:28.849362Z","shell.execute_reply":"2025-09-23T21:20:28.874925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"CATEGORY-SPECIFIC LIFECYCLE:\")\n\ncategory_lifecycle = (\n    final_stats\n    .join(articles.select(['article_id', 'product_group_name']), on='article_id', how='left')\n    .filter(pl.col('product_group_name').is_not_null())\n    .group_by('product_group_name')\n    .agg([\n        pl.count().alias('product_count'),\n        pl.col('half_life_weeks').mean().alias('avg_half_life'),\n        pl.col('half_life_weeks').median().alias('median_half_life'),\n        pl.col('half_life_weeks').std().alias('std_half_life'),\n        pl.col('peak_intensity').mean().alias('avg_peak_intensity'),\n        pl.col('weeks_active').mean().alias('avg_total_lifespan'),\n        pl.col('total_sales').mean().alias('avg_volume'),\n        \n        (pl.col('half_life_weeks') <= 4).mean().alias('fast_fashion_pct'),\n        (pl.col('half_life_weeks').is_between(5, 12)).mean().alias('seasonal_pct'),\n        (pl.col('half_life_weeks') > 12).mean().alias('staple_pct'),\n        \n        (pl.col('half_life_weeks') / pl.col('weeks_active')).mean().alias('decay_efficiency'),\n        pl.col('total_sales').sum().alias('total_category_volume')\n    ])\n    .filter(pl.col('product_count') >= 100)  # significant categories\n    .sort('median_half_life', descending=True)\n)\n\nprint(\"CATEGORY LIFECYCLE COMPARISON:\")\ncategory_lifecycle.to_pandas().round(2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:20:43.652306Z","iopub.execute_input":"2025-09-23T21:20:43.652740Z","iopub.status.idle":"2025-09-23T21:20:43.708172Z","shell.execute_reply.started":"2025-09-23T21:20:43.652703Z","shell.execute_reply":"2025-09-23T21:20:43.706749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"PRODUCT LIFECYCLE ARCHETYPES:\")\n\narchetypes = final_stats.with_columns([\n    pl.when(\n        (pl.col('half_life_weeks') <= 3) & (pl.col('peak_intensity') > 0.25)\n    ).then(pl.lit('Viral_Trends'))\n    .when(\n        (pl.col('half_life_weeks') <= 6) & (pl.col('peak_intensity') <= 0.25)\n    ).then(pl.lit('Fast_Fashion'))\n    .when(\n        (pl.col('half_life_weeks').is_between(7, 15)) & (pl.col('peak_intensity') <= 0.20)\n    ).then(pl.lit('Seasonal_Core'))\n    .when(\n        (pl.col('half_life_weeks') > 15) & (pl.col('peak_intensity') <= 0.15)\n    ).then(pl.lit('Evergreen_Basics'))\n    .otherwise(pl.lit('Mixed_Pattern'))\n    .alias('lifecycle_archetype')\n])\n\narchetype_summary = archetypes.group_by('lifecycle_archetype').agg([\n    pl.count().alias('count'),\n    pl.col('half_life_weeks').mean().alias('avg_half_life'),\n    pl.col('peak_intensity').mean().alias('avg_peak_intensity'),\n    pl.col('total_sales').mean().alias('avg_volume'),\n    pl.col('weeks_active').mean().alias('avg_lifespan')\n]).with_columns([\n    (pl.col('count') / archetypes.height * 100).alias('percentage')\n])\n\nprint(\"LIFECYCLE ARCHETYPES:\")\narchetype_summary.to_pandas().round(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:20:57.416563Z","iopub.execute_input":"2025-09-23T21:20:57.417733Z","iopub.status.idle":"2025-09-23T21:20:57.453177Z","shell.execute_reply.started":"2025-09-23T21:20:57.417678Z","shell.execute_reply":"2025-09-23T21:20:57.451786Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Observations: \n\n* Median popularity: 5 weeks\n* Average decay ratio: 34% -products lose half their popularity in 1/3 of their total lifespan.\n* 65% of products have short lifecycles (<6 weeks) confirms H&M's fast fashion model.\n* Upper body, lower body and accessories are core H&M categories with consistent patterns. (5 weeks median)\n* Nightwear: highest peak intensity (25%) + shortest half-life + seasonal/intimate purchasing.\n* Garment full body: trend-sensitive outerwear with quick obsolescence\n* Socks & Tights: highest variability (std=9.29) - mix of seasonal items (>6 weeks).\n* Fast Fashion (43%) - core business model - 3.75 weeks half-life\n* Viral Trends (%16) - ultra-short (2 weeks) with explosive peaks (38% intensity).\n* Seasonal Core (21%) - stable streams (9.83 weeks half-life)\n* Evergreen Basics (7%) - Long-term assets (>25 weeks)","metadata":{}},{"cell_type":"code","source":"half_life_features = final_stats.select([\n    'article_id',\n    'half_life_weeks',\n    'peak_intensity', \n    'total_sales',\n    'peak_sales',\n    'weeks_active',\n    'first_week',\n    'last_week',\n    'actual_fraction',\n    \n    (pl.col('half_life_weeks') / pl.col('weeks_active')).alias('lifecycle_efficiency_ratio'),\n    (pl.col('total_sales') / pl.col('weeks_active')).alias('sales_velocity'),\n    (pl.col('peak_sales') / pl.col('total_sales')).alias('peak_sales_ratio'),\n        \n    # TREND VELOCITY LABELING\n    pl.when(pl.col('half_life_weeks') <= 2)\n    .then(pl.lit('ultra_fast'))\n    .when(pl.col('half_life_weeks') <= 4)  \n    .then(pl.lit('fast_fashion'))         # core H&M model\n    .when(pl.col('half_life_weeks') <= 8)\n    .then(pl.lit('seasonal_trend'))       # season-driven items\n    .when(pl.col('half_life_weeks') <= 16)\n    .then(pl.lit('wardrobe_staple'))      # mid-season basics\n    .otherwise(pl.lit('timeless_basic'))  # year-round essentials\n    .alias('trend_velocity_category')\n\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:21:19.596039Z","iopub.execute_input":"2025-09-23T21:21:19.596400Z","iopub.status.idle":"2025-09-23T21:21:19.610144Z","shell.execute_reply.started":"2025-09-23T21:21:19.596377Z","shell.execute_reply":"2025-09-23T21:21:19.609101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"half_life_features.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:21:39.841012Z","iopub.execute_input":"2025-09-23T21:21:39.841334Z","iopub.status.idle":"2025-09-23T21:21:39.851095Z","shell.execute_reply.started":"2025-09-23T21:21:39.841313Z","shell.execute_reply":"2025-09-23T21:21:39.849962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# These are articles that were NOT in our half-life analysis.\nmature_article_ids = final_stats.get_column(\"article_id\").to_list()\nall_article_ids = articles.get_column(\"article_id\").to_list()\nnew_article_ids = set(all_article_ids) - set(mature_article_ids)\n\nprint(f\"\\nNumber of 'new' or 'low-interaction' articles: {len(new_article_ids)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:22:24.196477Z","iopub.execute_input":"2025-09-23T21:22:24.196832Z","iopub.status.idle":"2025-09-23T21:22:24.241205Z","shell.execute_reply.started":"2025-09-23T21:22:24.196808Z","shell.execute_reply":"2025-09-23T21:22:24.240031Z"}},"outputs":[],"execution_count":null}]}