{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":11000,"databundleVersionId":875412,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport time\n\n# --- 1. SETUP & CHARGEMENT DES DONNÉES PAR SEGMENTS ---\n\n# Chemin d'accès racine (selon la convention Kaggle)\nROOT_PATH = '../input/LANL-Earthquake-Prediction'\nTRAIN_FILE = 'train.csv'\n\n# Définition des paramètres de segmentation pour éviter la panne de mémoire\nSEGMENT_SIZE = 150000 \nCHUNK_SIZE = 150000 \nN_SEGMENTS = 4194 # Nombre total de segments de 150k\n\n# Initialisation du nouveau DataFrame pour stocker les features\nsegments_features = pd.DataFrame(index=range(N_SEGMENTS), \n                                 columns=['mean', 'std', 'max', 'min', 'time_to_failure'])\n\n# Types de données optimisés\nDTYPE = {'acoustic_data': np.int16, 'time_to_failure': np.float32}\n\nstart_time = time.time()\nprint(f\"Démarrage de l'extraction des {N_SEGMENTS} segments à partir de {TRAIN_FILE}...\")\n\n# Création de l'itérateur pour lire le fichier par morceaux\nchunk_iterator = pd.read_csv(os.path.join(ROOT_PATH, TRAIN_FILE), \n                             dtype=DTYPE, \n                             iterator=True, \n                             chunksize=CHUNK_SIZE)\n\ni = 0\nfor chunk in chunk_iterator:\n    # Récupérer les données acoustiques\n    acoustic_data = chunk['acoustic_data']\n    \n    # 1. Extraction de la valeur cible (Target) : dernière valeur du segment\n    target = chunk['time_to_failure'].iloc[-1]\n    \n    # 2. Extraction des caractéristiques de base\n    segments_features.loc[i, 'time_to_failure'] = target\n    segments_features.loc[i, 'mean'] = acoustic_data.mean()\n    segments_features.loc[i, 'std'] = acoustic_data.std()\n    segments_features.loc[i, 'max'] = acoustic_data.max()\n    segments_features.loc[i, 'min'] = acoustic_data.min()\n    \n    i += 1\n    # Affichage de la progression\n    if i % 1000 == 0:\n        print(f\"Progression : {i} segments traités / {N_SEGMENTS}\")\n\nend_time = time.time()\nprint(f\"Analyse des {N_SEGMENTS} segments terminée en {round(end_time - start_time, 2)} secondes.\")\n\n\n# --- 2. INSPECTION ET STATISTIQUES SUR LE NOUVEAU DATAFRAME ---\n\nprint(\"\\n--- Inspection du Nouveau DataFrame de Features ---\")\nprint(f\"Dimensions du DataFrame de features: {segments_features.shape}\")\nprint(\"Aperçu des 5 premières lignes du DataFrame de features:\")\nprint(segments_features.head())\n\nprint(\"\\n--- Statistiques Descriptives des Features (Segments) ---\")\nprint(segments_features.describe())\n\n\n# --- 3. CORRÉLATION ---\n\n# Calcul et affichage de la corrélation entre les features et la cible\nprint(\"\\n--- Corrélation de Pearson (Segment Features vs. Target) ---\")\ncorrelation_matrix = segments_features.corr()\nprint(correlation_matrix['time_to_failure'].sort_values(ascending=False))\n\n# Visualisation des corrélations (Heatmap)\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlation_matrix, annot=True, fmt=\".2f\", cmap='viridis')\nplt.title('Matrice de Corrélation des Features du Segment')\nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fonction pour calculer un ensemble plus riche de features pour un segment de 150k\ndef extract_advanced_features(chunk):\n    acoustic_data = chunk['acoustic_data']\n    \n    # 1. Caractéristiques de base (déjà calculées, mais incluses pour la complétude)\n    features = {\n        'mean': acoustic_data.mean(),\n        'std': acoustic_data.std(),\n    }\n    \n    # 2. Caractéristiques d'Amplitude/Énergie\n    features['abs_mean'] = np.mean(np.abs(acoustic_data))\n    features['abs_std'] = np.std(np.abs(acoustic_data))\n    features['rms'] = np.sqrt(np.mean(acoustic_data**2))\n    features['peak_to_peak'] = acoustic_data.max() - acoustic_data.min()\n\n    # 3. Caractéristiques Statistiques Avancées\n    features['skew'] = acoustic_data.skew()\n    features['kurt'] = acoustic_data.kurtosis()\n    features['q01'] = np.quantile(acoustic_data, 0.01)\n    features['q99'] = np.quantile(acoustic_data, 0.99)\n    \n    # 4. Caractéristiques de Fréquence (Approximation par Z-Crossings)\n    # Compte le nombre de fois où le signal passe d'une valeur positive à négative (ou vice versa)\n    features['z_crossings'] = ((acoustic_data.values[:-1] * acoustic_data.values[1:]) < 0).sum()\n    \n    return features\n\n# --- RE-EXECUTION AVEC LA NOUVELLE FONCTION ---\n\n# Redéfinition du DataFrame pour inclure les nouvelles features\nfeature_names = ['time_to_failure', 'mean', 'std', 'abs_mean', 'abs_std', \n                 'rms', 'peak_to_peak', 'skew', 'kurt', 'q01', 'q99', 'z_crossings']\nsegments_features_advanced = pd.DataFrame(index=range(N_SEGMENTS), columns=feature_names)\nDTYPE = {'acoustic_data': np.int16, 'time_to_failure': np.float32}\n\nchunk_iterator = pd.read_csv(os.path.join(ROOT_PATH, TRAIN_FILE), dtype=DTYPE, iterator=True, chunksize=CHUNK_SIZE)\n\ni = 0\nfor chunk in chunk_iterator:\n    # Récupération de la cible\n    segments_features_advanced.loc[i, 'time_to_failure'] = chunk['time_to_failure'].iloc[-1]\n    \n    # Calcul des features avancées\n    new_features = extract_advanced_features(chunk)\n    \n    for key, value in new_features.items():\n        segments_features_advanced.loc[i, key] = value\n        \n    i += 1\n    if i % 1000 == 0:\n        print(f\"Progression (Advanced Features): {i} segments traités / {N_SEGMENTS}\")\n\n# Affichage des corrélations des features avancées\nprint(\"\\n--- Corrélation des Features Avancées avec la Cible ---\")\nprint(segments_features_advanced.corr()['time_to_failure'].sort_values(ascending=False))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Conversion en types numériques pour s'assurer que le modèle fonctionne\nsegments_features_advanced = segments_features_advanced.apply(pd.to_numeric)\n\n# Séparation des features (X) et de la cible (y)\nX = segments_features_advanced.drop('time_to_failure', axis=1)\ny = segments_features_advanced['time_to_failure']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nimport os\nimport time\n\n# --- SETUP ---\nROOT_PATH = '../input/LANL-Earthquake-Prediction'\nTRAIN_FILE = 'train.csv'\nSEGMENT_SIZE = 150000\nCHUNK_SIZE = 150000\nN_SEGMENTS = 4194\nDTYPE = {'acoustic_data': np.int16, 'time_to_failure': np.float32}\n\n# --- 1. FONCTION D'INGÉNIERIE DES CARACTÉRISTIQUES AVANCÉES ---\ndef extract_advanced_features(chunk):\n    acoustic_data = chunk['acoustic_data'].values.astype(np.float32)\n\n    # a) Ajouter un petit bruit gaussien\n    noise = np.random.normal(0, 0.5, len(acoustic_data))\n    acoustic_data += noise\n\n    # b) Centrer par la médiane\n    acoustic_data -= np.median(acoustic_data)\n\n    features = {}\n\n    # --- Caractéristiques de base ---\n    features['mean'] = acoustic_data.mean()\n    features['std'] = acoustic_data.std()\n    features['max'] = acoustic_data.max()\n    features['min'] = acoustic_data.min()\n\n    # --- Amplitude / énergie ---\n    features['abs_mean'] = np.mean(np.abs(acoustic_data))\n    features['abs_std'] = np.std(np.abs(acoustic_data))\n    features['rms'] = np.sqrt(np.mean(acoustic_data**2))\n    features['peak_to_peak'] = acoustic_data.max() - acoustic_data.min()\n\n    # --- Statistiques avancées ---\n    features['skew'] = pd.Series(acoustic_data).skew()\n    features['kurt'] = pd.Series(acoustic_data).kurtosis()\n    features['q01'] = np.quantile(acoustic_data, 0.01)\n    features['q99'] = np.quantile(acoustic_data, 0.99)\n\n    # --- Fréquence approximative ---\n    features['z_crossings'] = ((acoustic_data[:-1] * acoustic_data[1:]) < 0).sum()\n\n    # --- Caractéristiques temporelles ---\n    diff = np.diff(acoustic_data)\n    features['mean_diff'] = diff.mean()\n    features['std_diff'] = diff.std()\n    # Rolling windows simples (ex: 100 points)\n    window = 100\n    features['rolling_std_mean'] = pd.Series(acoustic_data).rolling(window).std().mean()\n    features['rolling_mean_mean'] = pd.Series(acoustic_data).rolling(window).mean().mean()\n\n    return features\n\n# --- 2. EXTRACTION DES FEATURES SUR TOUS LES SEGMENTS ---\nfeature_names = [\n    'time_to_failure', 'mean', 'std', 'max', 'min', 'abs_mean', 'abs_std',\n    'rms', 'peak_to_peak', 'skew', 'kurt', 'q01', 'q99', 'z_crossings',\n    'mean_diff', 'std_diff', 'rolling_std_mean', 'rolling_mean_mean', 'cycle_id'\n]\n\nsegments_features_model = pd.DataFrame(index=range(N_SEGMENTS), columns=feature_names)\nchunk_iterator = pd.read_csv(os.path.join(ROOT_PATH, TRAIN_FILE), dtype=DTYPE,\n                             iterator=True, chunksize=CHUNK_SIZE)\n\ni = 0\nlast_ttf = 0\ncycle_count = 0\n\nfor chunk in chunk_iterator:\n    current_ttf = chunk['time_to_failure'].iloc[-1]\n\n    if current_ttf > last_ttf and last_ttf < 1:\n        cycle_count += 1\n\n    segments_features_model.loc[i, 'time_to_failure'] = current_ttf\n    segments_features_model.loc[i, 'cycle_id'] = cycle_count\n\n    new_features = extract_advanced_features(chunk)\n    for key, value in new_features.items():\n        segments_features_model.loc[i, key] = value\n\n    last_ttf = current_ttf\n    i += 1\n\n# Conversion numérique\nsegments_features_model = segments_features_model.apply(pd.to_numeric)\nprint(\"Extraction des features avancées terminée.\")\n\n# --- 3. FILTRAGE DES CYCLES POUR L'ENTRAÎNEMENT ---\nCYCLES_TO_KEEP = [1, 2, 3, 5, 7, 8, 10, 12, 14, 15]\nsegments_features_model = segments_features_model[\n    segments_features_model['cycle_id'].isin(CYCLES_TO_KEEP)\n]\n\n# --- 4. X / y ---\nX = segments_features_model.drop(['time_to_failure', 'cycle_id'], axis=1)\ny = segments_features_model['time_to_failure']\n\n# --- 5. ENTRAÎNEMENT LIGHTGBM ---\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=False)\nprint(f\"\\nDimensions du jeu d'entraînement filtré: {X_train.shape}\")\n\nparams = {\n    'objective': 'mae',\n    'metric': 'mae',\n    'n_estimators': 1000,\n    'learning_rate': 0.01,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 1,\n    'verbose': -1,\n    'n_jobs': -1,\n    'seed': 42\n}\n\nmodel = lgb.LGBMRegressor(**params)\n\nmodel.fit(\n    X_train, y_train,\n    eval_set=[(X_test, y_test)],\n    eval_metric='mae',\n    callbacks=[lgb.early_stopping(100, verbose=False)]\n)\n\n# --- 6. ÉVALUATION ---\ny_pred = model.predict(X_test)\nmae = mean_absolute_error(y_test, y_pred)\n\nprint(f\"\\nMAE sur le jeu de test filtré: {mae:.4f}\")\n\n# --- Importance des features ---\nimportance_df = pd.DataFrame({'feature': X.columns, 'importance': model.feature_importances_})\nprint(\"\\nTop 10 des features par importance:\")\nprint(importance_df.sort_values(by='importance', ascending=False).head(10))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport time\nimport lightgbm as lgb\n\n# --- SETUP ---\nROOT_PATH = '../input/LANL-Earthquake-Prediction'\nTEST_FOLDER = 'test'\nSUBMISSION_FILE = 'sample_submission.csv'\nSEGMENT_SIZE = 150000\nDTYPE_TEST = {'acoustic_data': np.int16}  # Le fichier test n'a pas de colonne time_to_failure\n\n# On suppose que le modèle LightGBM entraîné 'model' et le DataFrame X (pour les colonnes) sont accessibles.\n\n# --- 1. FONCTION DE FEATURE ENGINEERING ---\ndef extract_advanced_features(chunk):\n    acoustic_data = chunk['acoustic_data'].values\n    \n    # a) Ajouter un petit bruit gaussien constant (std=0.5)\n    noise = np.random.normal(0, 0.5, len(acoustic_data))\n    acoustic_data = acoustic_data + noise\n    \n    # b) Soustraire la médiane du segment\n    acoustic_data = acoustic_data - np.median(acoustic_data)\n    \n    features = {}\n    # Caractéristiques de base\n    features['mean'] = acoustic_data.mean()\n    features['std'] = acoustic_data.std()\n    features['rms'] = np.sqrt(np.mean(acoustic_data**2))\n    features['abs_mean'] = np.mean(np.abs(acoustic_data))\n    features['abs_std'] = np.std(np.abs(acoustic_data))\n    features['peak_to_peak'] = acoustic_data.max() - acoustic_data.min()\n    \n    # Caractéristiques statistiques\n    features['skew'] = pd.Series(acoustic_data).skew()\n    features['kurt'] = pd.Series(acoustic_data).kurtosis()\n    features['q01'] = np.quantile(acoustic_data, 0.01)\n    features['q99'] = np.quantile(acoustic_data, 0.99)\n    \n    # Fréquence approximative\n    features['z_crossings'] = ((acoustic_data[:-1] * acoustic_data[1:]) < 0).sum()\n    \n    # Dérivées\n    diff = np.diff(acoustic_data)\n    features['mean_diff'] = diff.mean()\n    features['std_diff'] = diff.std()\n    \n    # Rolling windows (taille = 1000)\n    window = 1000\n    series = pd.Series(acoustic_data)\n    features['rolling_mean_mean'] = series.rolling(window).mean().mean()\n    features['rolling_std_mean'] = series.rolling(window).std().mean()\n    \n    return features\n\n# --- 2. EXTRACTION DES FEATURES TEST ---\nsubmission = pd.read_csv(os.path.join(ROOT_PATH, SUBMISSION_FILE), index_col='seg_id')\n\n# Même colonnes que l’entraînement\nX_test_pred = pd.DataFrame(columns=X.columns, index=submission.index)\n\nprint(f\"Démarrage de l'extraction des features pour les {len(submission)} segments de test...\")\n\nstart_time = time.time()\ni = 0\n\nfor seg_id in X_test_pred.index:\n    file_path = os.path.join(ROOT_PATH, TEST_FOLDER, seg_id + '.csv')\n    segment_test = pd.read_csv(file_path, dtype=DTYPE_TEST)\n    \n    features = extract_advanced_features(segment_test)\n    \n    for feature_name, value in features.items():\n        if feature_name in X_test_pred.columns:\n            X_test_pred.loc[seg_id, feature_name] = value\n\n    i += 1\n    if i % 1000 == 0:\n        print(f\"Progression : {i} segments de test traités...\")\n\nend_time = time.time()\nprint(f\"Extraction des features de test terminée en {round(end_time - start_time, 2)} secondes.\")\n\n# --- 3. PREDICTION & SOUMISSION ---\nX_test_pred = X_test_pred.apply(pd.to_numeric)\nX_test_pred = X_test_pred.fillna(0)\n\nprint(\"\\nDémarrage de la prédiction...\")\npredictions = model.predict(X_test_pred)\n\nsubmission['time_to_failure'] = predictions\n\nprint(\"\\nAperçu du fichier de soumission final :\")\nprint(submission.head())\n\nsubmission.to_csv('submission_corrected.csv')\nprint(\"\\n--- Fichier submission_corrected.csv généré avec succès ! ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}