{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install lightgbm\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nimport tarfile\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestRegressor\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Install required packages if needed\n!pip install pyarrow\n\n# Load the parquet files\ntrain_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet')\nsample_submission = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet')\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Sample submission shape:\", sample_submission.shape)\nprint(\"\\nTrain columns:\", train_df.columns.tolist())\nprint(\"Sample submission columns:\", sample_submission.columns.tolist())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:37:36.506364Z","iopub.execute_input":"2025-06-23T04:37:36.506614Z","iopub.status.idle":"2025-06-23T04:38:14.491605Z","shell.execute_reply.started":"2025-06-23T04:37:36.506590Z","shell.execute_reply":"2025-06-23T04:38:14.488030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tarfile\nimport json\nimport os\nimport gc\n\n# Don't extract all files at once - process them one by one\ndef process_json_tar_efficiently(tar_path):\n    \"\"\"\n    Process JSON files from tar archive without extracting all to disk\n    \"\"\"\n    all_json_data = []\n    \n    with tarfile.open(tar_path, 'r') as tar:\n        # Get list of JSON files in the archive\n        json_members = [member for member in tar.getmembers() if member.name.endswith('.json')]\n        print(f\"Found {len(json_members)} JSON files in archive\")\n        \n        # Process files in batches to manage memory\n        batch_size = 10  # Process 10 files at a time\n        \n        for i in range(0, len(json_members), batch_size):\n            batch = json_members[i:i+batch_size]\n            print(f\"Processing batch {i//batch_size + 1}/{(len(json_members)-1)//batch_size + 1}\")\n            \n            for member in batch:\n                try:\n                    # Extract file to memory, not disk\n                    f = tar.extractfile(member)\n                    if f is not None:\n                        content = f.read().decode('utf-8')\n                        data = json.loads(content)\n                        \n                        # Process the JSON data\n                        processed_data = process_single_json(data, member.name)\n                        if processed_data:\n                            all_json_data.append(processed_data)\n                        \n                        # Clean up\n                        f.close()\n                        del content, data\n                        \n                except Exception as e:\n                    print(f\"Error processing {member.name}: {e}\")\n                    continue\n            \n            # Force garbage collection after each batch\n            gc.collect()\n            \n            # Stop after processing enough files for feature extraction\n            if len(all_json_data) >= 1000:  # Adjust this number based on your needs\n                print(f\"Processed {len(all_json_data)} files, stopping to save memory\")\n                break\n    \n    return all_json_data\n\ndef process_single_json(data, filename):\n    \"\"\"\n    Extract features from a single JSON object\n    Customize this based on your JSON structure\n    \"\"\"\n    try:\n        processed = {\n            'filename': filename,\n        }\n        \n        # Add feature extraction based on your JSON structure\n        if isinstance(data, dict):\n            # Example feature extractions - adjust based on your data\n            processed['num_keys'] = len(data.keys())\n            \n            # Common RecSys features\n            if 'user_id' in data:\n                processed['user_id'] = data['user_id']\n            if 'item_id' in data:\n                processed['item_id'] = data['item_id']\n            if 'rating' in data:\n                processed['rating'] = data['rating']\n            if 'timestamp' in data:\n                processed['timestamp'] = data['timestamp']\n            if 'interactions' in data:\n                processed['num_interactions'] = len(data['interactions']) if isinstance(data['interactions'], list) else 1\n            \n        return processed\n        \n    except Exception as e:\n        print(f\"Error processing JSON {filename}: {e}\")\n        return None\n\n# Process the JSON files efficiently\njson_data = process_json_tar_efficiently('/kaggle/input/aeroclub-recsys-2025/jsons_raw.tar.kaggle')\njson_df = pd.DataFrame(json_data)\n\nprint(f\"Extracted features from {len(json_df)} JSON files\")\nif not json_df.empty:\n    print(\"JSON DataFrame shape:\", json_df.shape)\n    print(\"JSON DataFrame columns:\", json_df.columns.tolist())\n    print(json_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:38:14.495172Z","iopub.execute_input":"2025-06-23T04:38:14.495674Z","iopub.status.idle":"2025-06-23T04:41:15.110971Z","shell.execute_reply.started":"2025-06-23T04:38:14.495632Z","shell.execute_reply":"2025-06-23T04:41:15.105917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data Exploration and Understanding\nprint(\"=== TRAIN DATA EXPLORATION ===\")\nprint(train_df.head())\nprint(\"\\nTrain data info:\")\nprint(train_df.info())\nprint(\"\\nTrain data description:\")\nprint(train_df.describe())\n\nprint(\"\\n=== SAMPLE SUBMISSION EXPLORATION ===\")\nprint(sample_submission.head())\nprint(\"\\nSample submission info:\")\nprint(sample_submission.info())\n\n# Check for missing values\nprint(\"\\n=== MISSING VALUES ===\")\nprint(\"Train missing values:\")\nprint(train_df.isnull().sum())\nprint(\"\\nSample submission missing values:\")\nprint(sample_submission.isnull().sum())\n\n# Initialize extracted_files as empty list since extraction failed\nextracted_files = []\n\n# Try to explore JSON structure without extracting files\nprint(\"\\n=== JSON STRUCTURE EXPLORATION ===\")\ntry:\n    # Read the JSON structure documentation\n    with open('/kaggle/input/aeroclub-recsys-2025/jsons_structure.md', 'r') as f:\n        json_structure = f.read()\n    print(\"JSON Structure Documentation:\")\n    print(json_structure)\nexcept Exception as e:\n    print(f\"Could not read JSON structure file: {e}\")\n\n# Try to peek at one JSON file without extracting all\nprint(\"\\n=== SAMPLE JSON PEEK ===\")\ntry:\n    import tarfile\n    import json\n    \n    with tarfile.open('/kaggle/input/aeroclub-recsys-2025/jsons_raw.tar.kaggle', 'r') as tar:\n        # Get first JSON file\n        json_members = [member for member in tar.getmembers() if member.name.endswith('.json')]\n        if json_members:\n            first_json = json_members[0]\n            f = tar.extractfile(first_json)\n            if f is not None:\n                content = f.read().decode('utf-8')\n                sample_json = json.loads(content)\n                print(f\"Sample JSON file: {first_json.name}\")\n                print(f\"JSON keys: {list(sample_json.keys()) if isinstance(sample_json, dict) else 'Not a dictionary'}\")\n                print(f\"JSON structure preview: {str(sample_json)[:500]}...\")\n                f.close()\n        else:\n            print(\"No JSON files found in archive\")\n            \nexcept Exception as e:\n    print(f\"Could not peek at JSON files: {e}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:41:15.113533Z","iopub.execute_input":"2025-06-23T04:41:15.113815Z","iopub.status.idle":"2025-06-23T04:44:32.747965Z","shell.execute_reply.started":"2025-06-23T04:41:15.113775Z","shell.execute_reply":"2025-06-23T04:44:32.743734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_recommendation_features(train_df):\n    \"\"\"\n    Create features for recommendation system using only parquet data\n    \"\"\"\n    df_features = train_df.copy()\n    \n    # Identify potential user/item columns\n    print(\"Available columns:\", df_features.columns.tolist())\n    \n    # Basic feature engineering based on common RecSys patterns\n    # Check if we have user_id column (or similar)\n    user_cols = [col for col in df_features.columns if 'user' in col.lower()]\n    item_cols = [col for col in df_features.columns if any(x in col.lower() for x in ['item', 'product', 'movie', 'book', 'song'])]\n    rating_cols = [col for col in df_features.columns if any(x in col.lower() for x in ['rating', 'score', 'target'])]\n    \n    print(f\"Detected user columns: {user_cols}\")\n    print(f\"Detected item columns: {item_cols}\")\n    print(f\"Detected rating columns: {rating_cols}\")\n    \n    # User-based features\n    if user_cols:\n        user_col = user_cols[0]  # Fix: Take first element, not the list\n        target_col = rating_cols[0] if rating_cols else df_features.columns[-1]  # Fix: Take first element\n        \n        user_stats = df_features.groupby(user_col).agg({\n            target_col: ['count', 'mean', 'std', 'min', 'max']\n        }).reset_index()\n        user_stats.columns = [user_col] + [f'user_{stat}' for stat in ['count', 'mean', 'std', 'min', 'max']]\n        df_features = df_features.merge(user_stats, on=user_col, how='left')\n        print(f\"Added user-based features for column: {user_col}\")\n    \n    # Item-based features\n    if item_cols:\n        item_col = item_cols[0]  # Fix: Take first element, not the list\n        target_col = rating_cols[0] if rating_cols else df_features.columns[-1]  # Fix: Take first element\n        \n        item_stats = df_features.groupby(item_col).agg({\n            target_col: ['count', 'mean', 'std', 'min', 'max']\n        }).reset_index()\n        item_stats.columns = [item_col] + [f'item_{stat}' for stat in ['count', 'mean', 'std', 'min', 'max']]\n        df_features = df_features.merge(item_stats, on=item_col, how='left')\n        print(f\"Added item-based features for column: {item_col}\")\n    \n    # Additional time-based features if timestamp exists\n    timestamp_cols = [col for col in df_features.columns if any(x in col.lower() for x in ['time', 'date', 'timestamp'])]\n    if timestamp_cols:\n        timestamp_col = timestamp_cols[0]  # Fix: Take first element, not the list\n        try:\n            df_features[timestamp_col] = pd.to_datetime(df_features[timestamp_col])\n            df_features['hour'] = df_features[timestamp_col].dt.hour\n            df_features['day_of_week'] = df_features[timestamp_col].dt.dayofweek\n            df_features['month'] = df_features[timestamp_col].dt.month\n            print(f\"Added time-based features for column: {timestamp_col}\")\n        except Exception as e:\n            print(f\"Could not convert {timestamp_col} to datetime: {e}\")\n    \n    return df_features\n\n# Apply feature engineering\ntrain_features = create_recommendation_features(train_df)\nprint(f\"Enhanced train features shape: {train_features.shape}\")\nprint(\"New columns added:\", [col for col in train_features.columns if col not in train_df.columns])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:44:32.749914Z","iopub.execute_input":"2025-06-23T04:44:32.750175Z","iopub.status.idle":"2025-06-23T04:44:49.220987Z","shell.execute_reply.started":"2025-06-23T04:44:32.750150Z","shell.execute_reply":"2025-06-23T04:44:49.215043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_features(df):\n    \"\"\"\n    Handle missing values and encode categorical variables\n    \"\"\"\n    df_processed = df.copy()\n    \n    # Fill missing values\n    numeric_columns = df_processed.select_dtypes(include=[np.number]).columns\n    categorical_columns = df_processed.select_dtypes(include=['object']).columns\n    \n    print(f\"Numeric columns: {len(numeric_columns)}\")\n    print(f\"Categorical columns: {len(categorical_columns)}\")\n    \n    for col in numeric_columns:\n        if df_processed[col].isnull().sum() > 0:\n            df_processed[col].fillna(df_processed[col].median(), inplace=True)\n            print(f\"Filled {col} missing values with median\")\n    \n    # Encode categorical variables\n    label_encoders = {}\n    for col in categorical_columns:\n        if df_processed[col].isnull().sum() > 0:\n            df_processed[col].fillna(df_processed[col].mode()[0] if len(df_processed[col].mode()) > 0 else 'unknown', inplace=True)\n        \n        le = LabelEncoder()\n        df_processed[col] = le.fit_transform(df_processed[col].astype(str))\n        label_encoders[col] = le\n        print(f\"Encoded categorical column: {col}\")\n    \n    return df_processed, label_encoders\n\n# Apply preprocessing\ntrain_processed, encoders = preprocess_features(train_features)\nprint(f\"Processed train shape: {train_processed.shape}\")\nprint(\"Preprocessing completed!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:44:49.222669Z","iopub.execute_input":"2025-06-23T04:44:49.222922Z","iopub.status.idle":"2025-06-23T04:50:57.146738Z","shell.execute_reply.started":"2025-06-23T04:44:49.222897Z","shell.execute_reply":"2025-06-23T04:50:57.141498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare features and target\n# Identify target column based on sample_submission\ntarget_col = sample_submission.columns[-1]  # Usually the last column is target\nif target_col not in train_processed.columns:\n    # If target column name doesn't match, use the last column of train data\n    target_col = train_processed.columns[-1]\n\nprint(f\"Target column: {target_col}\")\n\n# Prepare feature columns\nfeature_cols = [col for col in train_processed.columns if col != target_col]\nprint(f\"Number of features: {len(feature_cols)}\")\n\n# Check if we have valid features and target\nif len(feature_cols) == 0:\n    print(\"No feature columns found! Using all columns except the last one as features\")\n    feature_cols = train_processed.columns[:-1].tolist()\n    target_col = train_processed.columns[-1]\n\nX = train_processed[feature_cols]\ny = train_processed[target_col]\n\nprint(f\"Features shape: {X.shape}\")\nprint(f\"Target shape: {y.shape}\")\nprint(f\"Target statistics: {y.describe()}\")\n\n# Train-validation split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\nprint(f\"Training set: {X_train.shape}\")\nprint(f\"Validation set: {X_val.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:50:57.148796Z","iopub.execute_input":"2025-06-23T04:50:57.149018Z","iopub.status.idle":"2025-06-23T04:51:47.235843Z","shell.execute_reply.started":"2025-06-23T04:50:57.148996Z","shell.execute_reply":"2025-06-23T04:51:47.230377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check data types and identify problematic columns\nprint(\"Checking data types:\")\nprint(X_train.dtypes)\nprint(\"\\nProblematic columns:\")\ndatetime_cols = X_train.select_dtypes(include=['datetime64']).columns\nprint(\"Datetime columns:\", datetime_cols.tolist())\n\n# Fix datetime columns by converting to numeric\ndef fix_datetime_columns(df):\n    \"\"\"Convert datetime columns to numeric format\"\"\"\n    df_fixed = df.copy()\n    \n    # Convert datetime columns to timestamp (numeric)\n    datetime_cols = df_fixed.select_dtypes(include=['datetime64']).columns\n    for col in datetime_cols:\n        print(f\"Converting datetime column: {col}\")\n        # Convert to timestamp (seconds since epoch)\n        df_fixed[col] = pd.to_datetime(df_fixed[col]).astype('int64') // 10**9\n        \n    # Ensure all columns are numeric\n    for col in df_fixed.columns:\n        if df_fixed[col].dtype == 'object':\n            print(f\"Converting object column to numeric: {col}\")\n            df_fixed[col] = pd.to_numeric(df_fixed[col], errors='coerce')\n            df_fixed[col].fillna(0, inplace=True)\n    \n    return df_fixed\n\n# Apply fixes to training and validation data\nX_train_fixed = fix_datetime_columns(X_train)\nX_val_fixed = fix_datetime_columns(X_val)\n\nprint(\"Fixed data types:\")\nprint(X_train_fixed.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:51:47.237775Z","iopub.execute_input":"2025-06-23T04:51:47.237997Z","iopub.status.idle":"2025-06-23T04:52:01.961356Z","shell.execute_reply.started":"2025-06-23T04:51:47.237975Z","shell.execute_reply":"2025-06-23T04:52:01.957022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\n\n# LightGBM parameters\nlgb_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'random_state': 42\n}\n\n# Create datasets with fixed data\ntrain_data = lgb.Dataset(X_train_fixed, label=y_train)\nval_data = lgb.Dataset(X_val_fixed, label=y_val, reference=train_data)\n\n# Train model\nprint(\"Training LightGBM model...\")\nlgb_model = lgb.train(\n    lgb_params,\n    train_data,\n    valid_sets=[train_data, val_data],\n    num_boost_round=1000,\n    callbacks=[lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=100)]\n)\n\n# Validation predictions\nval_pred = lgb_model.predict(X_val_fixed, num_iteration=lgb_model.best_iteration)\nval_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\nprint(f\"Validation RMSE: {val_rmse:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:52:01.963745Z","iopub.execute_input":"2025-06-23T04:52:01.963981Z","iopub.status.idle":"2025-06-23T04:57:14.399971Z","shell.execute_reply.started":"2025-06-23T04:52:01.963958Z","shell.execute_reply":"2025-06-23T04:57:14.395310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create test features based on sample_submission structure\nprint(\"Preparing test data...\")\n\n# Check sample_submission structure\nprint(\"Sample submission columns:\", sample_submission.columns.tolist())\nprint(\"Sample submission shape:\", sample_submission.shape)\n\n# Method 1: If sample_submission contains features\nif len(sample_submission.columns) > 1:\n    # Extract features from sample_submission (excluding target column)\n    test_features = sample_submission.drop(columns=[sample_submission.columns[-1]])\n    \n    # Apply same feature engineering as training data\n    print(\"Applying feature engineering to test data...\")\n    test_enhanced = create_recommendation_features(test_features)\n    \n    # Apply same preprocessing\n    test_processed = test_enhanced.copy()\n    \n    # Handle categorical encoding for test data\n    categorical_columns = test_processed.select_dtypes(include=['object']).columns\n    for col in categorical_columns:\n        if col in encoders:\n            test_processed[col] = test_processed[col].astype(str)\n            # Handle unseen categories\n            unseen_mask = ~test_processed[col].isin(encoders[col].classes_)\n            if unseen_mask.any():\n                test_processed.loc[unseen_mask, col] = encoders[col].classes_[0]\n            test_processed[col] = encoders[col].transform(test_processed[col])\n    \n    # Fill missing values\n    numeric_columns = test_processed.select_dtypes(include=[np.number]).columns\n    for col in numeric_columns:\n        if test_processed[col].isnull().sum() > 0:\n            test_processed[col].fillna(test_processed[col].median(), inplace=True)\n    \n    # Ensure test data has same features as training data\n    missing_cols = set(feature_cols) - set(test_processed.columns)\n    for col in missing_cols:\n        test_processed[col] = 0\n    \n    # Select and reorder columns to match training features\n    test_processed = test_processed[feature_cols]\n    \n    print(f\"Test data shape after preprocessing: {test_processed.shape}\")\n    \n    # Generate predictions\n    test_pred = lgb_model.predict(test_processed, num_iteration=lgb_model.best_iteration)\n    \nelse:\n    # Method 2: If sample_submission only has ID and target columns\n    print(\"Sample submission appears to only have ID and target columns\")\n    print(\"You may need to create test features based on your competition requirements\")\n    \n    # Placeholder - replace with actual test data preparation logic\n    test_pred = np.random.random(len(sample_submission))\n    print(\"Using placeholder predictions - replace with actual test data logic\")\n\nprint(f\"Generated {len(test_pred)} test predictions\")\nprint(f\"Prediction statistics: min={test_pred.min():.4f}, max={test_pred.max():.4f}, mean={test_pred.mean():.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:57:14.401883Z","iopub.execute_input":"2025-06-23T04:57:14.402135Z","iopub.status.idle":"2025-06-23T04:57:30.771817Z","shell.execute_reply.started":"2025-06-23T04:57:14.402110Z","shell.execute_reply":"2025-06-23T04:57:30.767658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = sample_submission.copy()\n\n# Cast to integer (choose the right one based on your task)\nsubmission[submission.columns[-1]] = (test_pred >= 0.5).astype(int)  # or .astype(int), or np.round(...)\n\nprint(\"Submission preview:\")\nprint(submission.head(10))\n\nprint(f\"\\nSubmission shape: {submission.shape}\")\nprint(f\"Final prediction statistics:\")\nprint(submission[submission.columns[-1]].describe())\n\n# Save\nsubmission.to_parquet('submission.parquet', index=False)\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"Submission saved as 'submission.parquet' and 'submission.csv'\")\nprint(\"Ready for upload to Kaggle!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T04:57:30.774021Z","iopub.execute_input":"2025-06-23T04:57:30.774289Z","iopub.status.idle":"2025-06-23T04:57:51.471172Z","shell.execute_reply.started":"2025-06-23T04:57:30.774268Z","shell.execute_reply":"2025-06-23T04:57:51.465571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}