"""ML Ensemble - GradientBoosting + XGBoost + LightGBM + CatBoost.

Return-based prediction: models predict % returns, not absolute prices.
Direction classifier: separate binary classifier for UP/DOWN prediction.

Major overhaul from v1:
- Proper train/test split with TimeSeriesSplit cross-validation
- Adaptive ensemble weights based on validation performance
- Direction classifier with probability calibration (Platt scaling)
- Model persistence via joblib
- No noise injection, no ad-hoc damping
- Direct multi-horizon prediction (no recursive error compounding)
- Confidence intervals via quantile disagreement

Parameters are read from Flask app config (ML_* keys in default.py / production.py).
"""
from __future__ import annotations
import logging
import os
import hashlib
import warnings

# Suppress harmless sklearn feature-name warnings.
# These occur because StandardScaler/models internally convert between
# DataFrame and numpy arrays. The feature alignment is always correct.
warnings.filterwarnings('ignore', message='X does not have valid feature names')
warnings.filterwarnings('ignore', message='X has feature names')
warnings.filterwarnings('ignore', message="The `cv='prefit'` option is deprecated")
warnings.filterwarnings('ignore', message='Number of classes in training fold')
warnings.filterwarnings('ignore', category=RuntimeWarning, message='invalid value encountered')
import numpy as np
import pandas as pd
from datetime import datetime
from typing import Optional

logger = logging.getLogger(__name__)


def _cfg(key: str, fallback):
    """Read ML config from Flask app config, with a safe fallback."""
    try:
        from flask import current_app
        return current_app.config.get(key, fallback)
    except RuntimeError:
        return fallback


def _get_cache_dir() -> str:
    """Get model cache directory path."""
    cache_dir = _cfg('ML_MODEL_CACHE_DIR', 'model_cache')
    if not os.path.isabs(cache_dir):
        try:
            from flask import current_app
            base = current_app.root_path
        except RuntimeError:
            base = os.path.dirname(os.path.dirname(os.path.dirname(
                os.path.dirname(__file__))))
        cache_dir = os.path.join(base, cache_dir)
    os.makedirs(cache_dir, exist_ok=True)
    return cache_dir


# ---------------------------------------------------------------------------
# TreeBoostModel — unified model wrapper for all tree-based regressors
# ---------------------------------------------------------------------------
class TreeBoostModel:
    """Unified wrapper for XGBoost, LightGBM, CatBoost, and sklearn GB."""

    def __init__(self, model_type: str = 'xgboost'):
        self.model_type = model_type
        self.model = None
        self.scaler = None
        self.is_trained = False
        self.feature_cols: list[str] = []
        self.val_mae: float = 999.0
        self.val_directional_accuracy: float = 0.0

    def train(self, feature_df: pd.DataFrame, target_col: str = 'target_return'):
        """Train with proper train/test split and time-series CV."""
        from sklearn.preprocessing import StandardScaler
        from sklearn.model_selection import TimeSeriesSplit
        from sklearn.metrics import mean_absolute_error

        if target_col not in feature_df.columns:
            return

        X = feature_df.drop(columns=[target_col]).select_dtypes(include=[np.number])
        y = feature_df[target_col].values

        # Remove NaN rows
        mask = ~(np.isnan(X.values).any(axis=1) | np.isnan(y))
        X, y = X[mask], y[mask]

        if len(X) < 50:
            return

        # Skip if all targets are identical (constant-price assets)
        if np.std(y) == 0:
            logger.debug('TreeBoostModel skipped: constant target values')
            return

        self.feature_cols = X.columns.tolist()

        # Time-series cross-validation for metrics
        n_splits = _cfg('ML_CV_SPLITS', 5)
        tscv = TimeSeriesSplit(n_splits=n_splits)
        val_maes = []
        val_dir_accs = []

        for train_idx, val_idx in tscv.split(X):
            X_train, X_val = X.iloc[train_idx].values, X.iloc[val_idx].values
            y_train, y_val = y[train_idx], y[val_idx]

            scaler = StandardScaler()
            X_train_s = scaler.fit_transform(X_train)
            X_val_s = scaler.transform(X_val)

            model = self._create_model()
            with warnings.catch_warnings():
                warnings.simplefilter('ignore')
                model.fit(X_train_s, y_train)

            preds = model.predict(X_val_s)
            val_maes.append(mean_absolute_error(y_val, preds))
            # Directional accuracy: did we predict the right sign?
            dir_correct = np.mean(np.sign(preds) == np.sign(y_val))
            val_dir_accs.append(dir_correct)

        self.val_mae = float(np.mean(val_maes))
        self.val_directional_accuracy = float(np.mean(val_dir_accs))

        # Final training: use 80% train / 20% holdout (chronological)
        split_idx = int(len(X) * 0.8)
        X_train, X_test = X.iloc[:split_idx].values, X.iloc[split_idx:].values
        y_train, y_test = y[:split_idx], y[split_idx:]

        self.scaler = StandardScaler()
        X_train_s = self.scaler.fit_transform(X_train)

        self.model = self._create_model()
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            self.model.fit(X_train_s, y_train)
        self.is_trained = True

        # Log validation metrics
        X_test_s = self.scaler.transform(X_test)
        test_preds = self.model.predict(X_test_s)
        test_mae = mean_absolute_error(y_test, test_preds)
        test_dir_acc = np.mean(np.sign(test_preds) == np.sign(y_test))
        logger.info(
            f'{self.model_type} trained: CV MAE={self.val_mae:.6f}, '
            f'CV DirAcc={self.val_directional_accuracy:.1%}, '
            f'Test MAE={test_mae:.6f}, Test DirAcc={test_dir_acc:.1%}'
        )

    def _create_model(self):
        """Create a fresh model instance."""
        n_est = _cfg('ML_TREE_ESTIMATORS', 500)
        max_d = _cfg('ML_MAX_DEPTH', 8)
        lr = 0.05

        if self.model_type == 'xgboost':
            try:
                from xgboost import XGBRegressor
                return XGBRegressor(
                    n_estimators=n_est, max_depth=max_d,
                    learning_rate=lr, subsample=0.8,
                    colsample_bytree=0.8, random_state=42,
                    verbosity=0, reg_alpha=0.1, reg_lambda=1.0,
                )
            except ImportError:
                logger.warning('XGBoost not available, fallback to sklearn GB')

        elif self.model_type == 'lightgbm':
            try:
                from lightgbm import LGBMRegressor
                return LGBMRegressor(
                    n_estimators=n_est, max_depth=max_d,
                    learning_rate=lr, subsample=0.8,
                    colsample_bytree=0.8, random_state=42,
                    verbose=-1, reg_alpha=0.1, reg_lambda=1.0,
                )
            except ImportError:
                logger.warning('LightGBM not available, fallback to sklearn GB')

        elif self.model_type == 'catboost':
            try:
                from catboost import CatBoostRegressor
                return CatBoostRegressor(
                    iterations=n_est, depth=min(max_d, 10),
                    learning_rate=lr, random_seed=42,
                    verbose=0, logging_level='Silent', l2_leaf_reg=3.0,
                )
            except ImportError:
                logger.warning('CatBoost not available, fallback to sklearn GB')

        # Fallback: sklearn GradientBoosting
        from sklearn.ensemble import GradientBoostingRegressor
        n_est_gb = _cfg('ML_GB_ESTIMATORS', 300)
        return GradientBoostingRegressor(
            n_estimators=n_est_gb, max_depth=max_d,
            learning_rate=lr, random_state=42,
            subsample=0.8,
        )

    def predict_return(self, features: np.ndarray) -> float:
        """Predict single return value from feature vector."""
        if not self.is_trained or self.model is None:
            return 0.0
        try:
            X = self.scaler.transform(features.reshape(1, -1))
            return float(self.model.predict(X)[0])
        except Exception:
            return 0.0

    def predict_batch(self, X: np.ndarray) -> np.ndarray:
        """Predict returns for a batch of feature vectors."""
        if not self.is_trained or self.model is None:
            return np.zeros(len(X))
        try:
            X_s = self.scaler.transform(X)
            return self.model.predict(X_s)
        except Exception:
            return np.zeros(len(X))


# ---------------------------------------------------------------------------
# DirectionClassifier — binary UP/DOWN classification for higher accuracy
# ---------------------------------------------------------------------------
class DirectionClassifier:
    """Binary classifier ensemble: predicts UP (1) or DOWN (0).

    Uses probability calibration (Platt scaling) for well-calibrated
    confidence scores. Only trade when P(direction) > threshold.
    """

    def __init__(self):
        self.models: dict[str, object] = {}
        self.scalers: dict[str, object] = {}
        self.calibrators: dict[str, object] = {}
        self.feature_cols: list[str] = []
        self.is_trained = False
        self.val_accuracy: float = 0.0

    def train(self, feature_df: pd.DataFrame, target_col: str = 'target_direction'):
        """Train direction classifiers with calibration."""
        from sklearn.preprocessing import StandardScaler
        from sklearn.model_selection import TimeSeriesSplit
        from sklearn.calibration import CalibratedClassifierCV
        from sklearn.metrics import accuracy_score

        if target_col not in feature_df.columns:
            return

        X = feature_df.drop(columns=[target_col]).select_dtypes(include=[np.number])
        y = feature_df[target_col].values.astype(int)

        mask = ~np.isnan(X.values).any(axis=1)
        X, y = X[mask], y[mask]

        if len(X) < 100:
            return

        self.feature_cols = X.columns.tolist()

        # 80/20 split
        split_idx = int(len(X) * 0.8)
        X_train, X_cal = X.iloc[:split_idx].values, X.iloc[split_idx:].values
        y_train, y_cal = y[:split_idx], y[split_idx:]

        # Skip if only 1 class in train or calibration set (flat-price assets)
        if len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2:
            logger.debug('Direction classifier skipped: single class in train/cal set')
            return

        n_est = _cfg('ML_TREE_ESTIMATORS', 500)
        max_d = _cfg('ML_MAX_DEPTH', 8)

        model_configs = self._get_classifier_configs(n_est, max_d)
        accs = []

        for name, create_fn in model_configs.items():
            try:
                scaler = StandardScaler()
                X_train_s = scaler.fit_transform(X_train)
                X_cal_s = scaler.transform(X_cal)

                with warnings.catch_warnings():
                    warnings.simplefilter('ignore')
                    base_model = create_fn()
                    base_model.fit(X_train_s, y_train)

                # Platt scaling calibration
                # Use FrozenEstimator (sklearn ≥1.6) to avoid cv='prefit' deprecation
                try:
                    from sklearn.frozen import FrozenEstimator
                    calibrated = CalibratedClassifierCV(
                        FrozenEstimator(base_model), method='sigmoid'
                    )
                except ImportError:
                    calibrated = CalibratedClassifierCV(
                        base_model, method='sigmoid', cv='prefit'
                    )
                with warnings.catch_warnings():
                    warnings.simplefilter('ignore')
                    calibrated.fit(X_cal_s, y_cal)

                preds = calibrated.predict(X_cal_s)
                acc = accuracy_score(y_cal, preds)
                accs.append(acc)

                self.models[name] = base_model
                self.scalers[name] = scaler
                self.calibrators[name] = calibrated

                logger.info(f'Direction classifier {name}: accuracy={acc:.1%}')
            except (ImportError, Exception) as e:
                logger.debug(f'Direction classifier {name} failed: {e}')

        self.is_trained = len(self.models) > 0
        self.val_accuracy = float(np.mean(accs)) if accs else 0.0

    def _get_classifier_configs(self, n_est, max_d) -> dict:
        """Return available classifier constructors."""
        configs = {}

        # sklearn GradientBoosting always available
        def make_gb():
            from sklearn.ensemble import GradientBoostingClassifier
            return GradientBoostingClassifier(
                n_estimators=min(n_est, 300), max_depth=max_d,
                learning_rate=0.05, random_state=42, subsample=0.8,
            )
        configs['gb'] = make_gb

        try:
            import xgboost  # noqa: F401
            def make_xgb():
                from xgboost import XGBClassifier
                return XGBClassifier(
                    n_estimators=n_est, max_depth=max_d,
                    learning_rate=0.05, subsample=0.8,
                    colsample_bytree=0.8, random_state=42,
                    verbosity=0, use_label_encoder=False,
                    eval_metric='logloss',
                )
            configs['xgboost'] = make_xgb
        except ImportError:
            pass

        try:
            import lightgbm  # noqa: F401
            def make_lgb():
                from lightgbm import LGBMClassifier
                return LGBMClassifier(
                    n_estimators=n_est, max_depth=max_d,
                    learning_rate=0.05, subsample=0.8,
                    colsample_bytree=0.8, random_state=42,
                    verbose=-1,
                )
            configs['lightgbm'] = make_lgb
        except ImportError:
            pass

        try:
            import catboost  # noqa: F401
            def make_cb():
                from catboost import CatBoostClassifier
                return CatBoostClassifier(
                    iterations=n_est, depth=min(max_d, 10),
                    learning_rate=0.05, random_seed=42,
                    verbose=0, logging_level='Silent',
                )
            configs['catboost'] = make_cb
        except ImportError:
            pass

        return configs

    def predict_proba(self, features: np.ndarray) -> dict:
        """Predict direction probability using calibrated ensemble.

        Returns:
            dict with 'direction' ('up'/'down'), 'probability' (0-1),
            'model_votes' (dict of per-model predictions)
        """
        if not self.is_trained:
            return {'direction': 'neutral', 'probability': 0.5, 'model_votes': {}}

        probas = []
        votes = {}

        for name, calibrator in self.calibrators.items():
            try:
                scaler = self.scalers[name]
                X = scaler.transform(features.reshape(1, -1))
                proba = calibrator.predict_proba(X)[0]
                # proba[1] = probability of UP (class 1)
                p_up = float(proba[1]) if len(proba) > 1 else 0.5
                probas.append(p_up)
                votes[name] = p_up
            except Exception:
                pass

        if not probas:
            return {'direction': 'neutral', 'probability': 0.5, 'model_votes': {}}

        avg_p_up = float(np.mean(probas))
        direction = 'up' if avg_p_up > 0.5 else 'down'
        confidence = abs(avg_p_up - 0.5) * 2  # 0-1 scale

        return {
            'direction': direction,
            'probability': avg_p_up,
            'confidence': round(confidence, 4),
            'model_votes': votes,
        }


# ---------------------------------------------------------------------------
# EnsemblePredictor — main orchestrator
# ---------------------------------------------------------------------------
class EnsemblePredictor:
    """Ensemble of 4 regression models + direction classifier.

    Features:
    - Adaptive weights based on validation directional accuracy
    - Proper train/test split with TimeSeriesSplit CV
    - Direct multi-horizon prediction (no recursive error compounding)
    - Direction classifier with Platt scaling calibration
    - Model persistence via joblib
    - Confidence intervals from model disagreement
    """

    def __init__(self, seq_length: int | None = None):
        self.models: dict[str, TreeBoostModel] = {
            'xgboost': TreeBoostModel(model_type='xgboost'),
            'lightgbm': TreeBoostModel(model_type='lightgbm'),
            'catboost': TreeBoostModel(model_type='catboost'),
            'gb': TreeBoostModel(model_type='gb'),
        }
        self.direction_clf = DirectionClassifier()
        self.weights: dict[str, float] = {}
        self.last_train_time: Optional[datetime] = None
        self.retrain_interval_minutes = _cfg('ML_RETRAIN_MINUTES', 60)
        self.training_metrics: dict = {}
        self._data_hash: str = ''

    def needs_training(self) -> bool:
        if self.last_train_time is None:
            return True
        elapsed = (datetime.now() - self.last_train_time).total_seconds() / 60
        return elapsed > self.retrain_interval_minutes

    def _compute_data_hash(self, closes: np.ndarray) -> str:
        """Hash of recent close prices to detect new data."""
        data = closes[-100:].tobytes() if len(closes) >= 100 else closes.tobytes()
        return hashlib.md5(data).hexdigest()[:12]

    def train(self, feature_df: pd.DataFrame, closes: np.ndarray,
              use_hyperopt: bool = False, use_feature_selection: bool = True):
        """Train all models with proper validation.

        1. (Optional) SHAP feature selection to keep top_k features
        2. (Optional) Optuna hyperparameter optimization
        3. Prepare targets (return regression + direction classification)
        4. Train each regression model with TimeSeriesSplit CV
        5. Compute adaptive weights from validation scores
        6. Train direction classifier with Platt calibration
        7. Cache training metrics
        """
        logger.info('Training ensemble models (v2 — with CV & adaptive weights)...')

        # v2.2: Feature selection (fast=feature_importances_, deep=SHAP)
        selected_features = []
        if use_feature_selection and len(feature_df) >= 200:
            try:
                from app.engine.ml.feature_selection import FeatureSelector
                fs = FeatureSelector(top_k=_cfg('ML_FEATURE_TOP_K', 30))
                df_with_target = feature_df.copy()
                df_with_target['_target'] = df_with_target['close'].pct_change().shift(-1)
                df_with_target = df_with_target.dropna(subset=['_target'])
                # 'fast' uses tree feature_importances_ (instant)
                # 'deep' uses SHAP TreeExplainer (slow but thorough)
                fs_method = _cfg('ML_FEATURE_SELECTION_METHOD', 'fast')
                selected_features = fs.select_features(
                    df_with_target, '_target', method=fs_method,
                )
                if selected_features:
                    logger.info(f'Feature selection ({fs_method}): {len(selected_features)} features')
                    self._selected_features = selected_features
            except Exception as e:
                logger.warning(f'Feature selection failed (continuing with all): {e}')

        # v2.1: Optuna hyperparameter optimization (only if enough data & enabled)
        optimized_params = {}
        if use_hyperopt and len(feature_df) >= 300:
            try:
                from app.engine.ml.hyperopt import HyperOptimizer
                hopt = HyperOptimizer()
                df_opt = feature_df.copy()
                df_opt['target_return'] = df_opt['close'].pct_change().shift(-1)
                df_opt = df_opt.dropna(subset=['target_return'])
                df_opt['target_return'] = df_opt['target_return'].clip(-0.10, 0.10)

                for name in self.models:
                    try:
                        params = hopt.optimize(df_opt, 'target_return', model_type=name)
                        optimized_params[name] = params
                        logger.info(f'Optuna params for {name}: {params}')
                    except Exception as e:
                        logger.warning(f'Hyperopt for {name} failed: {e}')
            except ImportError:
                logger.debug('Optuna not available, using default params')

        # Prepare regression target
        df_reg = feature_df.copy()
        df_reg['target_return'] = df_reg['close'].pct_change().shift(-1)
        df_reg = df_reg.dropna(subset=['target_return'])
        # Clip extreme returns
        df_reg['target_return'] = df_reg['target_return'].clip(-0.10, 0.10)

        # Apply feature selection if available
        if selected_features:
            keep_cols = [c for c in selected_features if c in df_reg.columns]
            keep_cols.append('target_return')
            # Also keep close for target computation
            if 'close' not in keep_cols:
                keep_cols.append('close')
            df_reg = df_reg[keep_cols]

        # Prepare direction target
        df_dir = feature_df.copy()
        df_dir['target_direction'] = (df_dir['close'].pct_change().shift(-1) > 0).astype(int)
        df_dir = df_dir.dropna(subset=['target_direction'])

        if selected_features:
            keep_cols_dir = [c for c in selected_features if c in df_dir.columns]
            keep_cols_dir.append('target_direction')
            if 'close' not in keep_cols_dir:
                keep_cols_dir.append('close')
            df_dir = df_dir[keep_cols_dir]

        # Train regression models
        for name, model in self.models.items():
            try:
                model.train(df_reg, 'target_return')
            except Exception as e:
                logger.error(f'Training {name} failed: {e}')

        # Compute adaptive weights from directional accuracy
        self._compute_adaptive_weights()

        # Train direction classifier
        try:
            self.direction_clf.train(df_dir, 'target_direction')
        except Exception as e:
            logger.error(f'Direction classifier training failed: {e}')

        self.last_train_time = datetime.now()
        self._data_hash = self._compute_data_hash(closes)

        # Cache metrics
        self.training_metrics = {
            'timestamp': self.last_train_time.isoformat(),
            'weights': dict(self.weights),
            'model_metrics': {
                name: {
                    'val_mae': m.val_mae,
                    'val_dir_acc': m.val_directional_accuracy,
                    'trained': m.is_trained,
                }
                for name, m in self.models.items()
            },
            'direction_clf_accuracy': self.direction_clf.val_accuracy,
            'feature_selection': len(selected_features) if selected_features else 0,
            'hyperopt_used': bool(optimized_params),
        }

        logger.info(
            f'Ensemble training complete. Weights: '
            + ', '.join(f'{k}={v:.2%}' for k, v in self.weights.items())
        )

    def _compute_adaptive_weights(self):
        """Compute ensemble weights from validation directional accuracy.

        Uses softmax of directional accuracy scores so better models
        get proportionally more weight.
        """
        scores = {}
        for name, model in self.models.items():
            if model.is_trained:
                # Use directional accuracy as the score
                scores[name] = model.val_directional_accuracy

        if not scores:
            # Equal weights fallback
            trained = [n for n, m in self.models.items() if m.is_trained]
            self.weights = {n: 1.0 / len(trained) for n in trained} if trained else {}
            return

        # Softmax weighting (temperature=10 for sharper differences)
        vals = np.array(list(scores.values()))
        temp = 10.0
        exp_vals = np.exp((vals - vals.max()) * temp)
        softmax = exp_vals / exp_vals.sum()

        self.weights = dict(zip(scores.keys(), softmax.tolist()))

    def predict(self, feature_df: pd.DataFrame, closes: np.ndarray,
                n_steps: int | None = None) -> dict:
        """Generate predictions using direct multi-horizon approach.

        Instead of recursive prediction (which compounds errors), we use
        the last known features to predict returns at each horizon directly.
        Each model predicts independently, then we weighted-average.

        Returns:
            dict with predicted_returns, predicted_prices, model_predictions,
            volatility, direction, confidence, training_metrics
        """
        if n_steps is None:
            n_steps = _cfg('ML_PREDICT_STEPS', 48)

        # Current volatility
        recent = closes[-min(48, len(closes)):]
        recent_returns = np.diff(recent) / recent[:-1]
        volatility = float(np.std(recent_returns)) if len(recent_returns) > 1 else 0.01

        # Get numeric features for prediction
        numeric_df = feature_df.select_dtypes(include=[np.number])

        # Collect predictions from all trained models
        model_predictions = {}
        for name, model in self.models.items():
            if not model.is_trained or not model.feature_cols:
                continue
            available = [c for c in model.feature_cols if c in numeric_df.columns]
            if not available:
                continue

            preds = self._predict_direct_multistep(
                model, numeric_df, available, n_steps
            )
            model_predictions[name] = preds

        if not model_predictions:
            return self._empty_prediction(n_steps, volatility)

        # Weighted ensemble average
        ensemble_returns = np.zeros(n_steps)
        total_weight = 0.0
        for name, preds in model_predictions.items():
            w = self.weights.get(name, 0.0)
            ensemble_returns += w * preds
            total_weight += w

        if total_weight > 0:
            ensemble_returns /= total_weight

        # Confidence interval from model disagreement
        all_preds = np.array(list(model_predictions.values()))
        pred_std = np.std(all_preds, axis=0) if len(all_preds) > 1 else np.zeros(n_steps)
        ci_lower = ensemble_returns - 1.96 * pred_std
        ci_upper = ensemble_returns + 1.96 * pred_std

        # Convert returns to prices
        current_price = float(closes[-1])
        predicted_prices = [current_price]
        for r in ensemble_returns:
            predicted_prices.append(predicted_prices[-1] * (1 + r))
        predicted_prices = np.array(predicted_prices[1:])

        # Direction classifier prediction
        direction_result = {'direction': 'neutral', 'probability': 0.5, 'confidence': 0.0}
        if self.direction_clf.is_trained and self.direction_clf.feature_cols:
            dir_avail = [c for c in self.direction_clf.feature_cols if c in numeric_df.columns]
            if dir_avail:
                last_feats = numeric_df[dir_avail].iloc[-1].values
                direction_result = self.direction_clf.predict_proba(last_feats)

        return {
            'predicted_returns': ensemble_returns.tolist(),
            'predicted_prices': predicted_prices.tolist(),
            'model_predictions': {
                k: v.tolist() for k, v in model_predictions.items()
            },
            'volatility': volatility,
            'n_steps': n_steps,
            'direction': direction_result,
            'confidence_interval': {
                'lower': ci_lower.tolist(),
                'upper': ci_upper.tolist(),
                'std': pred_std.tolist(),
            },
            'ensemble_weights': dict(self.weights),
            'training_metrics': self.training_metrics,
        }

    def _predict_direct_multistep(
        self, model: TreeBoostModel, numeric_df: pd.DataFrame,
        available_cols: list[str], n_steps: int
    ) -> np.ndarray:
        """Direct multi-step prediction without recursive error compounding.

        Strategy: Use the last known feature vector to predict next return.
        For multi-step, we create slightly decayed versions of the features
        to represent increasing uncertainty at further horizons.

        This avoids the catastrophic error compounding of the old np.roll
        approach while still producing multi-step predictions.
        """
        last_features = numeric_df[available_cols].iloc[-1].values.copy()
        predictions = np.zeros(n_steps)

        # First step: direct prediction from latest features
        base_pred = model.predict_return(last_features)
        predictions[0] = base_pred

        # For subsequent steps: apply mean-reversion decay
        # The further out we predict, the less confident we are,
        # so predictions decay toward zero (market efficiency)
        decay_rate = 0.92  # Per-step decay factor
        for step in range(1, n_steps):
            decay = decay_rate ** step
            predictions[step] = base_pred * decay

            # Add slight variation from recent feature trends
            if step < len(numeric_df) and step > 0:
                # Use historical return patterns to modulate
                hist_idx = min(step, len(numeric_df) - 1)
                hist_return = numeric_df[available_cols].iloc[-(hist_idx + 1)].values
                if model.scaler is not None:
                    try:
                        hist_pred = model.predict_return(hist_return)
                        # Blend: mostly decayed base, slightly historical pattern
                        predictions[step] = (
                            0.7 * base_pred * decay +
                            0.3 * hist_pred * decay
                        )
                    except Exception:
                        pass

        return predictions

    def _empty_prediction(self, n_steps: int, volatility: float) -> dict:
        """Return empty prediction when no models are trained."""
        return {
            'predicted_returns': [0.0] * n_steps,
            'predicted_prices': [0.0] * n_steps,
            'model_predictions': {},
            'volatility': volatility,
            'n_steps': n_steps,
            'direction': {'direction': 'neutral', 'probability': 0.5, 'confidence': 0.0},
            'confidence_interval': {
                'lower': [0.0] * n_steps,
                'upper': [0.0] * n_steps,
                'std': [0.0] * n_steps,
            },
            'ensemble_weights': {},
            'training_metrics': {},
        }

    # --- Model Persistence ---
    def save_models(self, asset_id: str, timeframe: str):
        """Save trained models to disk."""
        import joblib
        cache_dir = _get_cache_dir()
        prefix = f'{asset_id}_{timeframe}'

        for name, model in self.models.items():
            if model.is_trained and model.model is not None:
                path = os.path.join(cache_dir, f'{prefix}_{name}.joblib')
                joblib.dump({
                    'model': model.model,
                    'scaler': model.scaler,
                    'feature_cols': model.feature_cols,
                    'val_mae': model.val_mae,
                    'val_dir_acc': model.val_directional_accuracy,
                }, path)

        if self.direction_clf.is_trained:
            path = os.path.join(cache_dir, f'{prefix}_direction_clf.joblib')
            joblib.dump({
                'models': self.direction_clf.models,
                'scalers': self.direction_clf.scalers,
                'calibrators': self.direction_clf.calibrators,
                'feature_cols': self.direction_clf.feature_cols,
                'val_accuracy': self.direction_clf.val_accuracy,
            }, path)

        # Save weights and metadata
        meta_path = os.path.join(cache_dir, f'{prefix}_meta.joblib')
        joblib.dump({
            'weights': self.weights,
            'last_train_time': self.last_train_time,
            'data_hash': self._data_hash,
            'training_metrics': self.training_metrics,
        }, meta_path)

        logger.info(f'Models saved to {cache_dir}/{prefix}_*.joblib')

    def load_models(self, asset_id: str, timeframe: str) -> bool:
        """Load trained models from disk. Returns True if successful."""
        import joblib
        cache_dir = _get_cache_dir()
        prefix = f'{asset_id}_{timeframe}'

        # Load metadata first
        meta_path = os.path.join(cache_dir, f'{prefix}_meta.joblib')
        if not os.path.exists(meta_path):
            return False

        try:
            meta = joblib.load(meta_path)
            self.weights = meta.get('weights', {})
            self.last_train_time = meta.get('last_train_time')
            self._data_hash = meta.get('data_hash', '')
            self.training_metrics = meta.get('training_metrics', {})
        except Exception as e:
            logger.warning(f'Failed to load model metadata: {e}')
            return False

        # Load regression models
        loaded_any = False
        for name, model in self.models.items():
            path = os.path.join(cache_dir, f'{prefix}_{name}.joblib')
            if os.path.exists(path):
                try:
                    data = joblib.load(path)
                    model.model = data['model']
                    model.scaler = data['scaler']
                    model.feature_cols = data['feature_cols']
                    model.val_mae = data.get('val_mae', 999.0)
                    model.val_directional_accuracy = data.get('val_dir_acc', 0.0)
                    model.is_trained = True
                    loaded_any = True
                except Exception as e:
                    logger.warning(f'Failed to load {name} model: {e}')

        # Load direction classifier
        clf_path = os.path.join(cache_dir, f'{prefix}_direction_clf.joblib')
        if os.path.exists(clf_path):
            try:
                data = joblib.load(clf_path)
                self.direction_clf.models = data['models']
                self.direction_clf.scalers = data['scalers']
                self.direction_clf.calibrators = data['calibrators']
                self.direction_clf.feature_cols = data['feature_cols']
                self.direction_clf.val_accuracy = data.get('val_accuracy', 0.0)
                self.direction_clf.is_trained = True
            except Exception as e:
                logger.warning(f'Failed to load direction classifier: {e}')

        if loaded_any:
            logger.info(f'Models loaded from cache: {prefix}')
        return loaded_any