042-数据分析项目实战

学习目标

通过本章节的学习，你将掌握：

• 数据分析项目的完整流程
• 数据获取、清洗和预处理技术
• 探索性数据分析(EDA)方法
• 数据可视化最佳实践
• 统计分析和机器学习应用
• 数据分析报告的编写
• 项目部署和自动化

1. 项目概述

1.1 项目背景

我们将构建一个完整的数据分析项目，以电商销售数据为例，分析用户行为、产品销售趋势、市场洞察等。

1.2 技术栈

# 核心数据分析库
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

# 机器学习库
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.cluster import KMeans
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import mean_squared_error, r2_score

# 统计分析
from scipy import stats
from scipy.stats import chi2_contingency, pearsonr
import statsmodels.api as sm
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.tsa.arima.model import ARIMA

# 数据库和API
import sqlite3
import requests
from sqlalchemy import create_engine

# 工具库
import warnings
import datetime as dt
from datetime import datetime, timedelta
import os
import json
from pathlib import Path

# 配置
warnings.filterwarnings('ignore')
plt.rcParams['font.sans-serif'] = ['SimHei']  # 中文字体
plt.rcParams['axes.unicode_minus'] = False    # 负号显示
sns.set_style("whitegrid")

print("数据分析环境配置完成")
print("主要库版本:")
print(f"- pandas: {pd.__version__}")
print(f"- numpy: {np.__version__}")
print(f"- matplotlib: {plt.matplotlib.__version__}")
print(f"- seaborn: {sns.__version__}")

1.3 项目结构

from pathlib import Path
import os

class ProjectStructure:
    """
    数据分析项目结构管理
    """
    
    def __init__(self, project_name: str = "ecommerce_analysis"):
        self.project_name = project_name
        self.base_path = Path(project_name)
        
    def create_structure(self):
        """
        创建项目目录结构
        """
        directories = [
            "data/raw",           # 原始数据
            "data/processed",     # 处理后数据
            "data/external",      # 外部数据
            "notebooks",          # Jupyter笔记本
            "src/data",          # 数据处理脚本
            "src/features",      # 特征工程
            "src/models",        # 模型代码
            "src/visualization", # 可视化代码
            "reports/figures",   # 图表输出
            "reports/tables",    # 表格输出
            "config",            # 配置文件
            "tests",             # 测试代码
            "docs",              # 文档
        ]
        
        for directory in directories:
            dir_path = self.base_path / directory
            dir_path.mkdir(parents=True, exist_ok=True)
            
            # 创建__init__.py文件
            if "src" in directory:
                (dir_path / "__init__.py").touch()
        
        # 创建配置文件
        self._create_config_files()
        
        # 创建README文件
        self._create_readme()
        
        print(f"项目结构创建完成: {self.project_name}")
        
    def _create_config_files(self):
        """
        创建配置文件
        """
        # requirements.txt
        requirements = [
            "pandas>=1.3.0",
            "numpy>=1.21.0",
            "matplotlib>=3.4.0",
            "seaborn>=0.11.0",
            "plotly>=5.0.0",
            "scikit-learn>=1.0.0",
            "scipy>=1.7.0",
            "statsmodels>=0.12.0",
            "jupyter>=1.0.0",
            "sqlalchemy>=1.4.0",
            "requests>=2.25.0",
            "python-dotenv>=0.19.0"
        ]
        
        with open(self.base_path / "requirements.txt", "w") as f:
            f.write("\n".join(requirements))
        
        # .env文件模板
        env_template = """
# 数据库配置
DATABASE_URL=sqlite:///data/ecommerce.db

# API配置
API_KEY=your_api_key_here
API_BASE_URL=https://api.example.com

# 项目配置
PROJECT_NAME=E-commerce Data Analysis
DATA_PATH=data/
OUTPUT_PATH=reports/

# 可视化配置
FIGURE_SIZE=(12, 8)
DPI=300
COLOR_PALETTE=viridis
        """
        
        with open(self.base_path / ".env.template", "w") as f:
            f.write(env_template.strip())
        
        # 配置文件
        config = {
            "data": {
                "raw_data_path": "data/raw/",
                "processed_data_path": "data/processed/",
                "external_data_path": "data/external/"
            },
            "analysis": {
                "test_size": 0.2,
                "random_state": 42,
                "cv_folds": 5
            },
            "visualization": {
                "figure_size": [12, 8],
                "dpi": 300,
                "style": "whitegrid",
                "palette": "viridis"
            },
            "output": {
                "reports_path": "reports/",
                "figures_path": "reports/figures/",
                "tables_path": "reports/tables/"
            }
        }
        
        with open(self.base_path / "config" / "config.json", "w") as f:
            json.dump(config, f, indent=2)
    
    def _create_readme(self):
        """
        创建README文件
        """
        readme_content = f"""
# {self.project_name.replace('_', ' ').title()}

## 项目概述

这是一个完整的数据分析项目，包含数据获取、清洗、分析、可视化和建模的完整流程。

## 项目结构

{self.project_name}/
├── data/
│ ├── raw/ # 原始数据
│ ├── processed/ # 处理后数据
│ └── external/ # 外部数据
├── notebooks/ # Jupyter笔记本
├── src/
│ ├── data/ # 数据处理
│ ├── features/ # 特征工程
│ ├── models/ # 模型代码
│ └── visualization/ # 可视化
├── reports/
│ ├── figures/ # 图表
│ └── tables/ # 表格
├── config/ # 配置文件
├── tests/ # 测试代码
└── docs/ # 文档

## 快速开始

1. 安装依赖：
```bash
pip install -r requirements.txt

2. 配置环境变量：

cp .env.template .env
# 编辑.env文件，填入实际配置

3. 运行分析：

jupyter notebook notebooks/

主要功能

• 数据获取和清洗
• 探索性数据分析
• 统计分析和假设检验
• 机器学习建模
• 数据可视化
• 自动化报告生成

技术栈

• Python 3.8+

• Pandas, NumPy

• Matplotlib, Seaborn, Plotly

• Scikit-learn

• Jupyter Notebook
  “”"

    with open(self.base_path / "README.md", "w") as f:
        f.write(readme_content.strip())
  

创建项目结构

project = ProjectStructure()
project.create_structure()

print(“\n项目结构创建完成！”)
print(“下一步：”)
print(“1. cd ecommerce_analysis”)
print(“2. pip install -r requirements.txt”)
print(“3. cp .env.template .env”)
print(“4. 开始数据分析之旅！”)

## 2. 数据获取与准备

### 2.1 数据生成器

```python
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import random
from faker import Faker

class EcommerceDataGenerator:
    """
    电商数据生成器
    """
    
    def __init__(self, seed: int = 42):
        self.fake = Faker('zh_CN')
        Faker.seed(seed)
        np.random.seed(seed)
        random.seed(seed)
        
    def generate_users(self, n_users: int = 10000) -> pd.DataFrame:
        """
        生成用户数据
        """
        users = []
        
        for i in range(n_users):
            user = {
                'user_id': f'U{i+1:06d}',
                'username': self.fake.user_name(),
                'email': self.fake.email(),
                'phone': self.fake.phone_number(),
                'gender': np.random.choice(['M', 'F'], p=[0.52, 0.48]),
                'age': np.random.normal(35, 12),
                'city': self.fake.city(),
                'province': self.fake.province(),
                'registration_date': self.fake.date_between(
                    start_date='-2y', end_date='today'
                ),
                'user_level': np.random.choice(
                    ['Bronze', 'Silver', 'Gold', 'Platinum'], 
                    p=[0.5, 0.3, 0.15, 0.05]
                )
            }
            
            # 年龄限制在18-70之间
            user['age'] = max(18, min(70, int(user['age'])))
            
            users.append(user)
        
        return pd.DataFrame(users)
    
    def generate_products(self, n_products: int = 1000) -> pd.DataFrame:
        """
        生成产品数据
        """
        categories = [
            '电子产品', '服装鞋帽', '家居用品', '美妆护肤', 
            '食品饮料', '图书音像', '运动户外', '母婴用品'
        ]
        
        brands = [
            '苹果', '华为', '小米', '三星', '耐克', '阿迪达斯',
            '优衣库', '宝洁', '联合利华', '雀巢', '可口可乐'
        ]
        
        products = []
        
        for i in range(n_products):
            category = np.random.choice(categories)
            
            # 根据类别设置价格范围
            if category == '电子产品':
                price_range = (100, 5000)
            elif category == '服装鞋帽':
                price_range = (50, 1000)
            elif category == '美妆护肤':
                price_range = (20, 500)
            else:
                price_range = (10, 300)
            
            product = {
                'product_id': f'P{i+1:06d}',
                'product_name': self.fake.catch_phrase(),
                'category': category,
                'brand': np.random.choice(brands),
                'price': np.random.uniform(*price_range),
                'cost': 0,  # 将在后面计算
                'stock': np.random.randint(0, 1000),
                'rating': np.random.normal(4.2, 0.8),
                'review_count': np.random.poisson(50),
                'launch_date': self.fake.date_between(
                    start_date='-3y', end_date='-1m'
                )
            }
            
            # 成本为价格的60-80%
            product['cost'] = product['price'] * np.random.uniform(0.6, 0.8)
            
            # 评分限制在1-5之间
            product['rating'] = max(1, min(5, product['rating']))
            
            products.append(product)
        
        return pd.DataFrame(products)
    
    def generate_orders(self, users_df: pd.DataFrame, 
                       products_df: pd.DataFrame, 
                       n_orders: int = 50000) -> pd.DataFrame:
        """
        生成订单数据
        """
        orders = []
        
        # 用户购买倾向（基于用户等级）
        user_purchase_prob = {
            'Bronze': 0.1,
            'Silver': 0.2, 
            'Gold': 0.4,
            'Platinum': 0.6
        }
        
        for i in range(n_orders):
            # 随机选择用户（考虑购买倾向）
            user_weights = users_df['user_level'].map(user_purchase_prob)
            user = users_df.sample(weights=user_weights).iloc[0]
            
            # 随机选择1-5个产品
            n_items = np.random.choice([1, 2, 3, 4, 5], p=[0.5, 0.25, 0.15, 0.07, 0.03])
            selected_products = products_df.sample(n_items)
            
            order_date = self.fake.date_between(
                start_date=max(user['registration_date'], 
                              datetime.now().date() - timedelta(days=365)),
                end_date='today'
            )
            
            total_amount = 0
            
            for _, product in selected_products.iterrows():
                quantity = np.random.choice([1, 2, 3], p=[0.7, 0.2, 0.1])
                
                # 添加一些随机折扣
                discount = np.random.choice([0, 0.1, 0.2, 0.3], p=[0.6, 0.2, 0.15, 0.05])
                unit_price = product['price'] * (1 - discount)
                
                order_item = {
                    'order_id': f'O{i+1:08d}',
                    'user_id': user['user_id'],
                    'product_id': product['product_id'],
                    'quantity': quantity,
                    'unit_price': unit_price,
                    'total_price': unit_price * quantity,
                    'discount': discount,
                    'order_date': order_date,
                    'order_status': np.random.choice(
                        ['completed', 'cancelled', 'returned'], 
                        p=[0.85, 0.1, 0.05]
                    ),
                    'payment_method': np.random.choice(
                        ['credit_card', 'alipay', 'wechat_pay', 'bank_transfer'],
                        p=[0.3, 0.4, 0.25, 0.05]
                    ),
                    'shipping_cost': np.random.choice([0, 10, 20], p=[0.3, 0.5, 0.2])
                }
                
                total_amount += order_item['total_price']
                orders.append(order_item)
        
        return pd.DataFrame(orders)
    
    def generate_user_behavior(self, users_df: pd.DataFrame,
                              products_df: pd.DataFrame,
                              n_events: int = 200000) -> pd.DataFrame:
        """
        生成用户行为数据
        """
        behaviors = []
        
        event_types = ['view', 'cart', 'purchase', 'favorite', 'share']
        event_weights = [0.6, 0.2, 0.1, 0.07, 0.03]
        
        for i in range(n_events):
            user = users_df.sample().iloc[0]
            product = products_df.sample().iloc[0]
            
            behavior = {
                'event_id': f'E{i+1:08d}',
                'user_id': user['user_id'],
                'product_id': product['product_id'],
                'event_type': np.random.choice(event_types, p=event_weights),
                'timestamp': self.fake.date_time_between(
                    start_date='-1y', end_date='now'
                ),
                'session_id': f'S{np.random.randint(1, 100000):06d}',
                'device_type': np.random.choice(
                    ['mobile', 'desktop', 'tablet'], 
                    p=[0.6, 0.3, 0.1]
                ),
                'source': np.random.choice(
                    ['direct', 'search', 'social', 'email', 'ads'],
                    p=[0.3, 0.25, 0.2, 0.15, 0.1]
                )
            }
            
            behaviors.append(behavior)
        
        return pd.DataFrame(behaviors)
    
    def generate_all_data(self, save_path: str = "data/raw/"):
        """
        生成所有数据并保存
        """
        print("开始生成数据...")
        
        # 生成各类数据
        users_df = self.generate_users(10000)
        products_df = self.generate_products(1000)
        orders_df = self.generate_orders(users_df, products_df, 50000)
        behaviors_df = self.generate_user_behavior(users_df, products_df, 200000)
        
        # 保存数据
        os.makedirs(save_path, exist_ok=True)
        
        users_df.to_csv(f"{save_path}/users.csv", index=False)
        products_df.to_csv(f"{save_path}/products.csv", index=False)
        orders_df.to_csv(f"{save_path}/orders.csv", index=False)
        behaviors_df.to_csv(f"{save_path}/user_behaviors.csv", index=False)
        
        print(f"数据生成完成！")
        print(f"- 用户数据: {len(users_df):,} 条")
        print(f"- 产品数据: {len(products_df):,} 条")
        print(f"- 订单数据: {len(orders_df):,} 条")
        print(f"- 行为数据: {len(behaviors_df):,} 条")
        
        return users_df, products_df, orders_df, behaviors_df

# 生成示例数据
if __name__ == "__main__":
    generator = EcommerceDataGenerator()
    users_df, products_df, orders_df, behaviors_df = generator.generate_all_data()
    
    print("\n数据预览:")
    print("\n用户数据:")
    print(users_df.head())
    print("\n产品数据:")
    print(products_df.head())
    print("\n订单数据:")
    print(orders_df.head())

2.2 数据加载和验证

class DataLoader:
    """
    数据加载和验证类
    """
    
    def __init__(self, data_path: str = "data/raw/"):
        self.data_path = data_path
        
    def load_data(self) -> dict:
        """
        加载所有数据文件
        """
        try:
            data = {
                'users': pd.read_csv(f"{self.data_path}/users.csv"),
                'products': pd.read_csv(f"{self.data_path}/products.csv"),
                'orders': pd.read_csv(f"{self.data_path}/orders.csv"),
                'behaviors': pd.read_csv(f"{self.data_path}/user_behaviors.csv")
            }
            
            # 数据类型转换
            data['users']['registration_date'] = pd.to_datetime(data['users']['registration_date'])
            data['products']['launch_date'] = pd.to_datetime(data['products']['launch_date'])
            data['orders']['order_date'] = pd.to_datetime(data['orders']['order_date'])
            data['behaviors']['timestamp'] = pd.to_datetime(data['behaviors']['timestamp'])
            
            print("数据加载成功！")
            return data
            
        except FileNotFoundError as e:
            print(f"数据文件未找到: {e}")
            print("请先运行数据生成器创建示例数据")
            return None
    
    def validate_data(self, data: dict) -> dict:
        """
        数据质量验证
        """
        validation_report = {}
        
        for table_name, df in data.items():
            report = {
                'shape': df.shape,
                'missing_values': df.isnull().sum().to_dict(),
                'duplicates': df.duplicated().sum(),
                'data_types': df.dtypes.to_dict()
            }
            
            # 特定验证
            if table_name == 'users':
                report['age_range'] = (df['age'].min(), df['age'].max())
                report['invalid_emails'] = (~df['email'].str.contains('@')).sum()
                
            elif table_name == 'products':
                report['price_range'] = (df['price'].min(), df['price'].max())
                report['negative_prices'] = (df['price'] < 0).sum()
                report['rating_range'] = (df['rating'].min(), df['rating'].max())
                
            elif table_name == 'orders':
                report['date_range'] = (df['order_date'].min(), df['order_date'].max())
                report['negative_amounts'] = (df['total_price'] < 0).sum()
                
            validation_report[table_name] = report
        
        return validation_report
    
    def print_validation_report(self, validation_report: dict):
        """
        打印验证报告
        """
        print("\n=== 数据质量验证报告 ===")
        
        for table_name, report in validation_report.items():
            print(f"\n{table_name.upper()} 表:")
            print(f"  数据形状: {report['shape']}")
            print(f"  重复行数: {report['duplicates']}")
            
            missing_values = {k: v for k, v in report['missing_values'].items() if v > 0}
            if missing_values:
                print(f"  缺失值: {missing_values}")
            else:
                print("  缺失值: 无")
            
            # 特定验证结果
            if 'age_range' in report:
                print(f"  年龄范围: {report['age_range']}")
                print(f"  无效邮箱: {report['invalid_emails']}")
            
            if 'price_range' in report:
                print(f"  价格范围: {report['price_range']}")
                print(f"  负价格数: {report['negative_prices']}")
                print(f"  评分范围: {report['rating_range']}")
            
            if 'date_range' in report:
                print(f"  日期范围: {report['date_range']}")
                print(f"  负金额数: {report['negative_amounts']}")

# 使用示例
loader = DataLoader()
data = loader.load_data()

if data:
    validation_report = loader.validate_data(data)
    loader.print_validation_report(validation_report)
    
    print("\n数据加载和验证完成！")
else:
    print("请先生成示例数据")

3. 数据清洗和预处理

3.1 数据清洗器

class DataCleaner:
    """
    数据清洗和预处理类
    """
    
    def __init__(self):
        self.cleaning_log = []
        
    def log_action(self, action: str, details: str = ""):
        """
        记录清洗操作
        """
        self.cleaning_log.append({
            'timestamp': datetime.now(),
            'action': action,
            'details': details
        })
    
    def clean_users_data(self, users_df: pd.DataFrame) -> pd.DataFrame:
        """
        清洗用户数据
        """
        df = users_df.copy()
        original_shape = df.shape
        
        # 1. 移除重复用户
        duplicates = df.duplicated(subset=['email']).sum()
        df = df.drop_duplicates(subset=['email'])
        self.log_action("移除重复用户", f"移除 {duplicates} 个重复邮箱")
        
        # 2. 清理年龄数据
        invalid_age = ((df['age'] < 18) | (df['age'] > 100)).sum()
        df = df[(df['age'] >= 18) & (df['age'] <= 100)]
        self.log_action("清理年龄数据", f"移除 {invalid_age} 个无效年龄")
        
        # 3. 标准化性别数据
        df['gender'] = df['gender'].map({'M': 'Male', 'F': 'Female'})
        
        # 4. 清理用户名
        df['username'] = df['username'].str.strip().str.lower()
        
        # 5. 验证邮箱格式
        email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
        invalid_emails = ~df['email'].str.match(email_pattern)
        df = df[~invalid_emails]
        self.log_action("验证邮箱格式", f"移除 {invalid_emails.sum()} 个无效邮箱")
        
        # 6. 添加衍生字段
        df['account_age_days'] = (datetime.now().date() - df['registration_date'].dt.date).dt.days
        df['age_group'] = pd.cut(df['age'], 
                                bins=[0, 25, 35, 45, 55, 100], 
                                labels=['18-25', '26-35', '36-45', '46-55', '55+'])
        
        print(f"用户数据清洗完成: {original_shape} -> {df.shape}")
        return df
    
    def clean_products_data(self, products_df: pd.DataFrame) -> pd.DataFrame:
        """
        清洗产品数据
        """
        df = products_df.copy()
        original_shape = df.shape
        
        # 1. 移除价格异常的产品
        invalid_price = ((df['price'] <= 0) | (df['price'] > 50000)).sum()
        df = df[(df['price'] > 0) & (df['price'] <= 50000)]
        self.log_action("清理价格数据", f"移除 {invalid_price} 个异常价格")
        
        # 2. 清理评分数据
        df['rating'] = df['rating'].clip(1, 5)
        
        # 3. 处理库存数据
        df['stock'] = df['stock'].fillna(0).astype(int)
        df['in_stock'] = df['stock'] > 0
        
        # 4. 标准化产品名称
        df['product_name'] = df['product_name'].str.strip()
        
        # 5. 添加衍生字段
        df['profit_margin'] = (df['price'] - df['cost']) / df['price']
        df['days_since_launch'] = (datetime.now().date() - df['launch_date'].dt.date).dt.days
        df['price_category'] = pd.cut(df['price'], 
                                     bins=[0, 50, 200, 1000, float('inf')], 
                                     labels=['低价', '中低价', '中高价', '高价'])
        
        print(f"产品数据清洗完成: {original_shape} -> {df.shape}")
        return df
    
    def clean_orders_data(self, orders_df: pd.DataFrame, 
                         users_df: pd.DataFrame, 
                         products_df: pd.DataFrame) -> pd.DataFrame:
        """
        清洗订单数据
        """
        df = orders_df.copy()
        original_shape = df.shape
        
        # 1. 移除无效订单
        invalid_amount = (df['total_price'] <= 0).sum()
        df = df[df['total_price'] > 0]
        self.log_action("移除无效订单", f"移除 {invalid_amount} 个零金额订单")
        
        # 2. 验证用户和产品ID的有效性
        valid_users = set(users_df['user_id'])
        valid_products = set(products_df['product_id'])
        
        invalid_user_orders = ~df['user_id'].isin(valid_users)
        invalid_product_orders = ~df['product_id'].isin(valid_products)
        
        df = df[~invalid_user_orders & ~invalid_product_orders]
        self.log_action("验证关联ID", 
                       f"移除 {invalid_user_orders.sum()} 个无效用户订单, "
                       f"{invalid_product_orders.sum()} 个无效产品订单")
        
        # 3. 处理数量异常
        df = df[(df['quantity'] > 0) & (df['quantity'] <= 10)]
        
        # 4. 添加衍生字段
        df['order_month'] = df['order_date'].dt.to_period('M')
        df['order_weekday'] = df['order_date'].dt.day_name()
        df['order_hour'] = df['order_date'].dt.hour
        
        # 5. 计算订单级别统计
        order_stats = df.groupby('order_id').agg({
            'total_price': 'sum',
            'quantity': 'sum',
            'product_id': 'count'
        }).rename(columns={'product_id': 'item_count'})
        
        df = df.merge(order_stats, on='order_id', suffixes=('', '_order'))
        
        print(f"订单数据清洗完成: {original_shape} -> {df.shape}")
        return df
    
    def clean_behaviors_data(self, behaviors_df: pd.DataFrame,
                           users_df: pd.DataFrame,
                           products_df: pd.DataFrame) -> pd.DataFrame:
        """
        清洗用户行为数据
        """
        df = behaviors_df.copy()
        original_shape = df.shape
        
        # 1. 验证用户和产品ID
        valid_users = set(users_df['user_id'])
        valid_products = set(products_df['product_id'])
        
        valid_behaviors = (df['user_id'].isin(valid_users) & 
                          df['product_id'].isin(valid_products))
        df = df[valid_behaviors]
        
        # 2. 移除重复行为
        duplicates = df.duplicated().sum()
        df = df.drop_duplicates()
        self.log_action("移除重复行为", f"移除 {duplicates} 个重复记录")
        
        # 3. 添加时间相关字段
        df['date'] = df['timestamp'].dt.date
        df['hour'] = df['timestamp'].dt.hour
        df['weekday'] = df['timestamp'].dt.day_name()
        
        # 4. 创建会话统计
        session_stats = df.groupby('session_id').agg({
            'event_id': 'count',
            'timestamp': ['min', 'max']
        })
        session_stats.columns = ['session_events', 'session_start', 'session_end']
        session_stats['session_duration'] = (
            session_stats['session_end'] - session_stats['session_start']
        ).dt.total_seconds() / 60  # 分钟
        
        df = df.merge(session_stats[['session_events', 'session_duration']], 
                     on='session_id')
        
        print(f"行为数据清洗完成: {original_shape} -> {df.shape}")
        return df
    
    def clean_all_data(self, data: dict) -> dict:
        """
        清洗所有数据
        """
        print("开始数据清洗...")
        
        cleaned_data = {}
        
        # 按依赖顺序清洗
        cleaned_data['users'] = self.clean_users_data(data['users'])
        cleaned_data['products'] = self.clean_products_data(data['products'])
        cleaned_data['orders'] = self.clean_orders_data(
            data['orders'], 
            cleaned_data['users'], 
            cleaned_data['products']
        )
        cleaned_data['behaviors'] = self.clean_behaviors_data(
            data['behaviors'],
            cleaned_data['users'],
            cleaned_data['products']
        )
        
        print("\n数据清洗完成！")
        self.print_cleaning_log()
        
        return cleaned_data
    
    def print_cleaning_log(self):
        """
        打印清洗日志
        """
        print("\n=== 数据清洗日志 ===")
        for log_entry in self.cleaning_log:
            print(f"{log_entry['timestamp'].strftime('%H:%M:%S')} - "
                  f"{log_entry['action']}: {log_entry['details']}")
    
    def save_cleaned_data(self, cleaned_data: dict, save_path: str = "data/processed/"):
        """
        保存清洗后的数据
        """
        os.makedirs(save_path, exist_ok=True)
        
        for table_name, df in cleaned_data.items():
            file_path = f"{save_path}/{table_name}_cleaned.csv"
            df.to_csv(file_path, index=False)
            print(f"已保存: {file_path}")
        
        # 保存清洗日志
        log_df = pd.DataFrame(self.cleaning_log)
        log_df.to_csv(f"{save_path}/cleaning_log.csv", index=False)
        print(f"已保存清洗日志: {save_path}/cleaning_log.csv")

# 使用示例
if data:
    cleaner = DataCleaner()
    cleaned_data = cleaner.clean_all_data(data)
    cleaner.save_cleaned_data(cleaned_data)
else:
    print("请先加载数据")

4. 探索性数据分析(EDA)

4.1 基础统计分析

class EDAAnalyzer:
    """
    探索性数据分析器
    """
    
    def __init__(self, data: dict):
        self.data = data
        self.figures_path = "reports/figures/"
        os.makedirs(self.figures_path, exist_ok=True)
        
    def basic_statistics(self):
        """
        基础统计信息
        """
        print("=== 基础统计信息 ===")
        
        for table_name, df in self.data.items():
            print(f"\n{table_name.upper()} 表统计:")
            print(f"数据形状: {df.shape}")
            
            # 数值型字段统计
            numeric_cols = df.select_dtypes(include=[np.number]).columns
            if len(numeric_cols) > 0:
                print("\n数值型字段统计:")
                print(df[numeric_cols].describe())
            
            # 分类型字段统计
            categorical_cols = df.select_dtypes(include=['object']).columns
            if len(categorical_cols) > 0:
                print("\n分类型字段统计:")
                for col in categorical_cols[:3]:  # 只显示前3个
                    print(f"\n{col}:")
                    print(df[col].value_counts().head())
    
    def user_analysis(self):
        """
        用户分析
        """
        users_df = self.data['users']
        
        print("\n=== 用户分析 ===")
        
        # 1. 年龄分布
        plt.figure(figsize=(15, 10))
        
        plt.subplot(2, 3, 1)
        plt.hist(users_df['age'], bins=20, alpha=0.7, color='skyblue')
        plt.title('用户年龄分布')
        plt.xlabel('年龄')
        plt.ylabel('用户数')
        
        # 2. 性别分布
        plt.subplot(2, 3, 2)
        gender_counts = users_df['gender'].value_counts()
        plt.pie(gender_counts.values, labels=gender_counts.index, autopct='%1.1f%%')
        plt.title('用户性别分布')
        
        # 3. 用户等级分布
        plt.subplot(2, 3, 3)
        level_counts = users_df['user_level'].value_counts()
        plt.bar(level_counts.index, level_counts.values, color='lightgreen')
        plt.title('用户等级分布')
        plt.xticks(rotation=45)
        
        # 4. 注册时间趋势
        plt.subplot(2, 3, 4)
        registration_trend = users_df.groupby(
            users_df['registration_date'].dt.to_period('M')
        ).size()
        registration_trend.plot(kind='line')
        plt.title('用户注册趋势')
        plt.xticks(rotation=45)
        
        # 5. 年龄组分布
        plt.subplot(2, 3, 5)
        age_group_counts = users_df['age_group'].value_counts()
        plt.bar(age_group_counts.index, age_group_counts.values, color='orange')
        plt.title('年龄组分布')
        plt.xticks(rotation=45)
        
        # 6. 账户年龄分布
        plt.subplot(2, 3, 6)
        plt.hist(users_df['account_age_days'], bins=20, alpha=0.7, color='purple')
        plt.title('账户年龄分布(天)')
        plt.xlabel('天数')
        plt.ylabel('用户数')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/user_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 统计摘要
        print(f"总用户数: {len(users_df):,}")
        print(f"平均年龄: {users_df['age'].mean():.1f} 岁")
        print(f"男女比例: {users_df['gender'].value_counts(normalize=True).to_dict()}")
        print(f"平均账户年龄: {users_df['account_age_days'].mean():.0f} 天")
    
    def product_analysis(self):
        """
        产品分析
        """
        products_df = self.data['products']
        
        print("\n=== 产品分析 ===")
        
        plt.figure(figsize=(15, 10))
        
        # 1. 价格分布
        plt.subplot(2, 3, 1)
        plt.hist(products_df['price'], bins=30, alpha=0.7, color='skyblue')
        plt.title('产品价格分布')
        plt.xlabel('价格')
        plt.ylabel('产品数')
        
        # 2. 类别分布
        plt.subplot(2, 3, 2)
        category_counts = products_df['category'].value_counts()
        plt.pie(category_counts.values, labels=category_counts.index, autopct='%1.1f%%')
        plt.title('产品类别分布')
        
        # 3. 评分分布
        plt.subplot(2, 3, 3)
        plt.hist(products_df['rating'], bins=20, alpha=0.7, color='lightgreen')
        plt.title('产品评分分布')
        plt.xlabel('评分')
        plt.ylabel('产品数')
        
        # 4. 价格vs评分
        plt.subplot(2, 3, 4)
        plt.scatter(products_df['price'], products_df['rating'], alpha=0.5)
        plt.title('价格 vs 评分')
        plt.xlabel('价格')
        plt.ylabel('评分')
        
        # 5. 利润率分布
        plt.subplot(2, 3, 5)
        plt.hist(products_df['profit_margin'], bins=20, alpha=0.7, color='orange')
        plt.title('利润率分布')
        plt.xlabel('利润率')
        plt.ylabel('产品数')
        
        # 6. 库存状态
        plt.subplot(2, 3, 6)
        stock_status = products_df['in_stock'].value_counts()
        plt.bar(['缺货', '有库存'], stock_status.values, color=['red', 'green'])
        plt.title('库存状态')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/product_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 统计摘要
        print(f"总产品数: {len(products_df):,}")
        print(f"平均价格: ¥{products_df['price'].mean():.2f}")
        print(f"价格范围: ¥{products_df['price'].min():.2f} - ¥{products_df['price'].max():.2f}")
        print(f"平均评分: {products_df['rating'].mean():.2f}")
        print(f"平均利润率: {products_df['profit_margin'].mean():.1%}")
        print(f"有库存产品比例: {products_df['in_stock'].mean():.1%}")

4.2 销售分析

    def sales_analysis(self):
        """
        销售分析
        """
        orders_df = self.data['orders']
        
        print("\n=== 销售分析 ===")
        
        # 只分析已完成的订单
        completed_orders = orders_df[orders_df['order_status'] == 'completed']
        
        plt.figure(figsize=(20, 12))
        
        # 1. 销售额趋势
        plt.subplot(3, 4, 1)
        daily_sales = completed_orders.groupby('order_date')['total_price'].sum()
        daily_sales.plot(kind='line')
        plt.title('日销售额趋势')
        plt.xticks(rotation=45)
        
        # 2. 月销售额
        plt.subplot(3, 4, 2)
        monthly_sales = completed_orders.groupby('order_month')['total_price'].sum()
        monthly_sales.plot(kind='bar')
        plt.title('月销售额')
        plt.xticks(rotation=45)
        
        # 3. 订单状态分布
        plt.subplot(3, 4, 3)
        status_counts = orders_df['order_status'].value_counts()
        plt.pie(status_counts.values, labels=status_counts.index, autopct='%1.1f%%')
        plt.title('订单状态分布')
        
        # 4. 支付方式分布
        plt.subplot(3, 4, 4)
        payment_counts = completed_orders['payment_method'].value_counts()
        plt.bar(payment_counts.index, payment_counts.values)
        plt.title('支付方式分布')
        plt.xticks(rotation=45)
        
        # 5. 订单金额分布
        plt.subplot(3, 4, 5)
        plt.hist(completed_orders['total_price'], bins=30, alpha=0.7)
        plt.title('订单金额分布')
        plt.xlabel('金额')
        plt.ylabel('订单数')
        
        # 6. 星期几销售模式
        plt.subplot(3, 4, 6)
        weekday_sales = completed_orders.groupby('order_weekday')['total_price'].sum()
        weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        weekday_sales = weekday_sales.reindex(weekday_order)
        weekday_sales.plot(kind='bar')
        plt.title('星期销售模式')
        plt.xticks(rotation=45)
        
        # 7. 小时销售模式
        plt.subplot(3, 4, 7)
        hourly_sales = completed_orders.groupby('order_hour')['total_price'].sum()
        hourly_sales.plot(kind='line', marker='o')
        plt.title('小时销售模式')
        plt.xlabel('小时')
        
        # 8. 订单数量分布
        plt.subplot(3, 4, 8)
        quantity_dist = completed_orders['quantity'].value_counts().sort_index()
        plt.bar(quantity_dist.index, quantity_dist.values)
        plt.title('订单数量分布')
        plt.xlabel('数量')
        
        # 9. 折扣分布
        plt.subplot(3, 4, 9)
        discount_dist = completed_orders['discount'].value_counts().sort_index()
        plt.bar(discount_dist.index, discount_dist.values)
        plt.title('折扣分布')
        plt.xlabel('折扣率')
        
        # 10. 运费分布
        plt.subplot(3, 4, 10)
        shipping_dist = completed_orders['shipping_cost'].value_counts().sort_index()
        plt.bar(shipping_dist.index, shipping_dist.values)
        plt.title('运费分布')
        plt.xlabel('运费')
        
        # 11. 订单项目数分布
        plt.subplot(3, 4, 11)
        items_dist = completed_orders['item_count'].value_counts().sort_index()
        plt.bar(items_dist.index, items_dist.values)
        plt.title('订单项目数分布')
        plt.xlabel('项目数')
        
        # 12. 客单价趋势
        plt.subplot(3, 4, 12)
        daily_avg_order = completed_orders.groupby('order_date')['total_price_order'].mean()
        daily_avg_order.plot(kind='line')
        plt.title('客单价趋势')
        plt.xticks(rotation=45)
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/sales_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 统计摘要
        total_revenue = completed_orders['total_price'].sum()
        total_orders = len(completed_orders)
        avg_order_value = completed_orders['total_price_order'].mean()
        
        print(f"总销售额: ¥{total_revenue:,.2f}")
        print(f"总订单数: {total_orders:,}")
        print(f"平均客单价: ¥{avg_order_value:.2f}")
        print(f"订单完成率: {len(completed_orders)/len(orders_df):.1%}")
        print(f"平均订单项目数: {completed_orders['item_count'].mean():.1f}")
    
    def customer_analysis(self):
        """
        客户分析
        """
        users_df = self.data['users']
        orders_df = self.data['orders']
        
        print("\n=== 客户分析 ===")
        
        # 客户购买行为分析
        completed_orders = orders_df[orders_df['order_status'] == 'completed']
        
        # 计算客户指标
        customer_metrics = completed_orders.groupby('user_id').agg({
            'order_id': 'nunique',  # 订单数
            'total_price': 'sum',   # 总消费
            'order_date': ['min', 'max'],  # 首次和最后购买时间
            'quantity': 'sum'       # 总购买数量
        })
        
        customer_metrics.columns = ['order_count', 'total_spent', 'first_order', 'last_order', 'total_quantity']
        customer_metrics['avg_order_value'] = customer_metrics['total_spent'] / customer_metrics['order_count']
        customer_metrics['days_between_orders'] = (
            customer_metrics['last_order'] - customer_metrics['first_order']
        ).dt.days
        
        # 合并用户信息
        customer_analysis = users_df.merge(customer_metrics, left_on='user_id', right_index=True, how='left')
        customer_analysis = customer_analysis.fillna(0)
        
        plt.figure(figsize=(15, 10))
        
        # 1. 客户价值分布
        plt.subplot(2, 3, 1)
        plt.hist(customer_analysis[customer_analysis['total_spent'] > 0]['total_spent'], 
                bins=30, alpha=0.7)
        plt.title('客户总消费分布')
        plt.xlabel('总消费金额')
        plt.ylabel('客户数')
        
        # 2. 订单频次分布
        plt.subplot(2, 3, 2)
        order_freq = customer_analysis[customer_analysis['order_count'] > 0]['order_count']
        plt.hist(order_freq, bins=range(1, int(order_freq.max())+2), alpha=0.7)
        plt.title('客户订单频次分布')
        plt.xlabel('订单数')
        plt.ylabel('客户数')
        
        # 3. 客户等级vs消费
        plt.subplot(2, 3, 3)
        level_spending = customer_analysis.groupby('user_level')['total_spent'].mean()
        plt.bar(level_spending.index, level_spending.values)
        plt.title('不同等级客户平均消费')
        plt.xticks(rotation=45)
        
        # 4. 年龄vs消费
        plt.subplot(2, 3, 4)
        age_spending = customer_analysis.groupby('age_group')['total_spent'].mean()
        plt.bar(age_spending.index, age_spending.values)
        plt.title('不同年龄组平均消费')
        plt.xticks(rotation=45)
        
        # 5. 性别vs消费
        plt.subplot(2, 3, 5)
        gender_spending = customer_analysis.groupby('gender')['total_spent'].mean()
        plt.bar(gender_spending.index, gender_spending.values)
        plt.title('不同性别平均消费')
        
        # 6. 客单价分布
        plt.subplot(2, 3, 6)
        avg_order_values = customer_analysis[customer_analysis['avg_order_value'] > 0]['avg_order_value']
        plt.hist(avg_order_values, bins=30, alpha=0.7)
        plt.title('客单价分布')
        plt.xlabel('平均订单金额')
        plt.ylabel('客户数')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/customer_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # RFM分析
        self.rfm_analysis(customer_analysis)
        
        # 统计摘要
        active_customers = len(customer_analysis[customer_analysis['order_count'] > 0])
        print(f"活跃客户数: {active_customers:,}")
        print(f"客户活跃率: {active_customers/len(customer_analysis):.1%}")
        print(f"平均客户价值: ¥{customer_analysis['total_spent'].mean():.2f}")
        print(f"平均订单频次: {customer_analysis['order_count'].mean():.1f}")
        print(f"平均客单价: ¥{customer_analysis['avg_order_value'].mean():.2f}")

4.3 RFM客户分析

    def rfm_analysis(self, customer_data: pd.DataFrame):
        """
        RFM客户分析
        """
        print("\n=== RFM客户分析 ===")
        
        # 只分析有购买行为的客户
        active_customers = customer_data[customer_data['order_count'] > 0].copy()
        
        if len(active_customers) == 0:
            print("没有活跃客户数据")
            return
        
        # 计算RFM指标
        reference_date = active_customers['last_order'].max()
        
        # Recency: 最近一次购买距今天数
        active_customers['recency'] = (reference_date - active_customers['last_order']).dt.days
        
        # Frequency: 购买频次
        active_customers['frequency'] = active_customers['order_count']
        
        # Monetary: 总消费金额
        active_customers['monetary'] = active_customers['total_spent']
        
        # RFM评分（1-5分，5分最好）
        active_customers['R_score'] = pd.qcut(active_customers['recency'], 5, labels=[5,4,3,2,1])
        active_customers['F_score'] = pd.qcut(active_customers['frequency'].rank(method='first'), 5, labels=[1,2,3,4,5])
        active_customers['M_score'] = pd.qcut(active_customers['monetary'], 5, labels=[1,2,3,4,5])
        
        # 组合RFM分数
        active_customers['RFM_score'] = (
            active_customers['R_score'].astype(str) + 
            active_customers['F_score'].astype(str) + 
            active_customers['M_score'].astype(str)
        )
        
        # 客户分群
        def segment_customers(row):
            if row['RFM_score'] in ['555', '554', '544', '545', '454', '455', '445']:
                return '冠军客户'
            elif row['RFM_score'] in ['543', '444', '435', '355', '354', '345', '344', '335']:
                return '忠实客户'
            elif row['RFM_score'] in ['512', '511', '422', '421', '412', '411', '311']:
                return '潜力客户'
            elif row['RFM_score'] in ['155', '154', '144', '214', '215', '115', '114']:
                return '新客户'
            elif row['RFM_score'] in ['533', '532', '531', '523', '522', '521', '515', '514', '513', '425', '424', '413', '414', '415', '315', '314', '313']:
                return '需要关注'
            elif row['RFM_score'] in ['331', '321', '231', '241', '251']:
                return '即将流失'
            elif row['RFM_score'] in ['155', '154', '144', '214', '215', '115', '114']:
                return '高价值流失'
            else:
                return '已流失客户'
        
        active_customers['customer_segment'] = active_customers.apply(segment_customers, axis=1)
        
        # 可视化RFM分析
        plt.figure(figsize=(15, 10))
        
        # 1. RFM分布
        plt.subplot(2, 3, 1)
        plt.hist(active_customers['recency'], bins=20, alpha=0.7, color='red')
        plt.title('Recency分布')
        plt.xlabel('天数')
        
        plt.subplot(2, 3, 2)
        plt.hist(active_customers['frequency'], bins=20, alpha=0.7, color='green')
        plt.title('Frequency分布')
        plt.xlabel('订单数')
        
        plt.subplot(2, 3, 3)
        plt.hist(active_customers['monetary'], bins=20, alpha=0.7, color='blue')
        plt.title('Monetary分布')
        plt.xlabel('消费金额')
        
        # 2. 客户分群
        plt.subplot(2, 3, 4)
        segment_counts = active_customers['customer_segment'].value_counts()
        plt.pie(segment_counts.values, labels=segment_counts.index, autopct='%1.1f%%')
        plt.title('客户分群分布')
        
        # 3. RFM热力图
        plt.subplot(2, 3, 5)
        rfm_summary = active_customers.groupby(['R_score', 'F_score']).size().unstack(fill_value=0)
        sns.heatmap(rfm_summary, annot=True, fmt='d', cmap='YlOrRd')
        plt.title('RF热力图')
        
        # 4. 分群价值分析
        plt.subplot(2, 3, 6)
        segment_value = active_customers.groupby('customer_segment')['monetary'].mean().sort_values(ascending=False)
        plt.bar(range(len(segment_value)), segment_value.values)
        plt.xticks(range(len(segment_value)), segment_value.index, rotation=45)
        plt.title('各分群平均价值')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/rfm_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 分群统计
        segment_stats = active_customers.groupby('customer_segment').agg({
            'user_id': 'count',
            'recency': 'mean',
            'frequency': 'mean', 
            'monetary': 'mean'
        }).round(2)
        segment_stats.columns = ['客户数', '平均间隔天数', '平均订单数', '平均消费']
        
        print("\n客户分群统计:")
        print(segment_stats)
        
        return active_customers
    
    def behavior_analysis(self):
        """
        用户行为分析
        """
        behaviors_df = self.data['behaviors']
        
        print("\n=== 用户行为分析 ===")
        
        plt.figure(figsize=(15, 10))
        
        # 1. 行为类型分布
        plt.subplot(2, 3, 1)
        event_counts = behaviors_df['event_type'].value_counts()
        plt.pie(event_counts.values, labels=event_counts.index, autopct='%1.1f%%')
        plt.title('用户行为类型分布')
        
        # 2. 设备类型分布
        plt.subplot(2, 3, 2)
        device_counts = behaviors_df['device_type'].value_counts()
        plt.bar(device_counts.index, device_counts.values)
        plt.title('设备类型分布')
        
        # 3. 流量来源分布
        plt.subplot(2, 3, 3)
        source_counts = behaviors_df['source'].value_counts()
        plt.bar(source_counts.index, source_counts.values)
        plt.title('流量来源分布')
        plt.xticks(rotation=45)
        
        # 4. 小时行为模式
        plt.subplot(2, 3, 4)
        hourly_behavior = behaviors_df['hour'].value_counts().sort_index()
        plt.plot(hourly_behavior.index, hourly_behavior.values, marker='o')
        plt.title('小时行为模式')
        plt.xlabel('小时')
        plt.ylabel('行为数')
        
        # 5. 星期行为模式
        plt.subplot(2, 3, 5)
        weekday_behavior = behaviors_df['weekday'].value_counts()
        weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        weekday_behavior = weekday_behavior.reindex(weekday_order)
        plt.bar(weekday_behavior.index, weekday_behavior.values)
        plt.title('星期行为模式')
        plt.xticks(rotation=45)
        
        # 6. 会话时长分布
        plt.subplot(2, 3, 6)
        session_duration = behaviors_df['session_duration'].dropna()
        plt.hist(session_duration[session_duration < 60], bins=30, alpha=0.7)
        plt.title('会话时长分布(分钟)')
        plt.xlabel('时长')
        plt.ylabel('会话数')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/behavior_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 行为转化分析
        self.conversion_analysis(behaviors_df)
        
        # 统计摘要
        print(f"总行为数: {len(behaviors_df):,}")
        print(f"独立用户数: {behaviors_df['user_id'].nunique():,}")
        print(f"独立会话数: {behaviors_df['session_id'].nunique():,}")
        print(f"平均会话时长: {behaviors_df['session_duration'].mean():.1f} 分钟")
        print(f"平均会话行为数: {behaviors_df['session_events'].mean():.1f}")
    
    def conversion_analysis(self, behaviors_df: pd.DataFrame):
        """
        转化漏斗分析
        """
        print("\n=== 转化漏斗分析 ===")
        
        # 计算各步骤用户数
        funnel_data = {
            '浏览': behaviors_df[behaviors_df['event_type'] == 'view']['user_id'].nunique(),
            '加购物车': behaviors_df[behaviors_df['event_type'] == 'cart']['user_id'].nunique(),
            '收藏': behaviors_df[behaviors_df['event_type'] == 'favorite']['user_id'].nunique(),
            '购买': behaviors_df[behaviors_df['event_type'] == 'purchase']['user_id'].nunique(),
            '分享': behaviors_df[behaviors_df['event_type'] == 'share']['user_id'].nunique()
        }
        
        # 计算转化率
        total_users = funnel_data['浏览']
        conversion_rates = {k: v/total_users for k, v in funnel_data.items()}
        
        # 可视化漏斗
        plt.figure(figsize=(12, 8))
        
        plt.subplot(1, 2, 1)
        steps = list(funnel_data.keys())
        values = list(funnel_data.values())
        plt.bar(steps, values, color=['skyblue', 'lightgreen', 'orange', 'red', 'purple'])
        plt.title('转化漏斗 - 用户数')
        plt.ylabel('用户数')
        plt.xticks(rotation=45)
        
        # 添加数值标签
        for i, v in enumerate(values):
            plt.text(i, v + max(values)*0.01, f'{v:,}', ha='center')
        
        plt.subplot(1, 2, 2)
        rates = list(conversion_rates.values())
        plt.bar(steps, rates, color=['skyblue', 'lightgreen', 'orange', 'red', 'purple'])
        plt.title('转化漏斗 - 转化率')
        plt.ylabel('转化率')
        plt.xticks(rotation=45)
        
        # 添加百分比标签
        for i, v in enumerate(rates):
            plt.text(i, v + max(rates)*0.01, f'{v:.1%}', ha='center')
        
        plt.tight_layout()
        plt.savefig(f"{self.figures_path}/conversion_funnel.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        print("转化漏斗统计:")
        for step, count in funnel_data.items():
            rate = conversion_rates[step]
            print(f"{step}: {count:,} 用户 ({rate:.1%})")
    
    def run_full_analysis(self):
        """
        运行完整的EDA分析
        """
        print("开始探索性数据分析...")
        
        self.basic_statistics()
        self.user_analysis()
        self.product_analysis()
        self.sales_analysis()
        self.customer_analysis()
        self.behavior_analysis()
        
        print("\n=== EDA分析完成 ===")
        print(f"所有图表已保存到: {self.figures_path}")

# 使用示例
if 'cleaned_data' in locals():
    eda = EDAAnalyzer(cleaned_data)
    eda.run_full_analysis()
else:
    print("请先完成数据清洗")

5. 统计分析和假设检验

5.1 统计分析器

class StatisticalAnalyzer:
    """
    统计分析和假设检验
    """
    
    def __init__(self, data: dict):
        self.data = data
        self.results = {}
        
    def correlation_analysis(self):
        """
        相关性分析
        """
        print("=== 相关性分析 ===")
        
        # 用户数据相关性
        users_df = self.data['users']
        user_numeric = users_df.select_dtypes(include=[np.number])
        
        plt.figure(figsize=(15, 5))
        
        # 用户数据相关性热力图
        plt.subplot(1, 3, 1)
        user_corr = user_numeric.corr()
        sns.heatmap(user_corr, annot=True, cmap='coolwarm', center=0)
        plt.title('用户数据相关性')
        
        # 产品数据相关性
        products_df = self.data['products']
        product_numeric = products_df.select_dtypes(include=[np.number])
        
        plt.subplot(1, 3, 2)
        product_corr = product_numeric.corr()
        sns.heatmap(product_corr, annot=True, cmap='coolwarm', center=0)
        plt.title('产品数据相关性')
        
        # 订单数据相关性
        orders_df = self.data['orders']
        order_numeric = orders_df.select_dtypes(include=[np.number])
        
        plt.subplot(1, 3, 3)
        order_corr = order_numeric.corr()
        sns.heatmap(order_corr, annot=True, cmap='coolwarm', center=0)
        plt.title('订单数据相关性')
        
        plt.tight_layout()
        plt.savefig("reports/figures/correlation_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 保存相关性结果
        self.results['correlations'] = {
            'users': user_corr,
            'products': product_corr,
            'orders': order_corr
        }
        
        # 强相关性分析
        print("\n强相关性分析 (|r| > 0.5):")
        for data_type, corr_matrix in self.results['correlations'].items():
            print(f"\n{data_type.upper()}:")
            strong_corr = corr_matrix[(abs(corr_matrix) > 0.5) & (corr_matrix != 1.0)]
            strong_corr = strong_corr.dropna(how='all').dropna(axis=1, how='all')
            if not strong_corr.empty:
                print(strong_corr)
            else:
                print("无强相关性")
    
    def hypothesis_testing(self):
        """
        假设检验
        """
        print("\n=== 假设检验 ===")
        
        users_df = self.data['users']
        orders_df = self.data['orders']
        products_df = self.data['products']
        
        # 1. 性别与消费金额的关系
        print("\n1. 性别与消费金额关系检验")
        
        # 合并用户和订单数据
        user_orders = orders_df.merge(users_df[['user_id', 'gender']], on='user_id')
        completed_orders = user_orders[user_orders['order_status'] == 'completed']
        
        male_spending = completed_orders[completed_orders['gender'] == 'Male']['total_price']
        female_spending = completed_orders[completed_orders['gender'] == 'Female']['total_price']
        
        # t检验
        t_stat, p_value = stats.ttest_ind(male_spending, female_spending)
        
        print(f"男性平均消费: ¥{male_spending.mean():.2f}")
        print(f"女性平均消费: ¥{female_spending.mean():.2f}")
        print(f"t统计量: {t_stat:.4f}")
        print(f"p值: {p_value:.4f}")
        print(f"结论: {'存在显著差异' if p_value < 0.05 else '无显著差异'}")
        
        self.results['gender_spending_test'] = {
            't_statistic': t_stat,
            'p_value': p_value,
            'significant': p_value < 0.05
        }
        
        # 2. 年龄组与消费金额的关系
        print("\n2. 年龄组与消费金额关系检验(ANOVA)")
        
        user_orders_age = completed_orders.merge(
            users_df[['user_id', 'age_group']], on='user_id'
        )
        
        age_groups = []
        for group in user_orders_age['age_group'].unique():
            if pd.notna(group):
                group_data = user_orders_age[user_orders_age['age_group'] == group]['total_price']
                age_groups.append(group_data)
        
        if len(age_groups) > 1:
            f_stat, p_value = stats.f_oneway(*age_groups)
            
            print(f"F统计量: {f_stat:.4f}")
            print(f"p值: {p_value:.4f}")
            print(f"结论: {'年龄组间存在显著差异' if p_value < 0.05 else '年龄组间无显著差异'}")
            
            self.results['age_spending_anova'] = {
                'f_statistic': f_stat,
                'p_value': p_value,
                'significant': p_value < 0.05
            }
        
        # 3. 产品价格与评分的相关性检验
        print("\n3. 产品价格与评分相关性检验")
        
        price_rating_corr, p_value = pearsonr(products_df['price'], products_df['rating'])
        
        print(f"相关系数: {price_rating_corr:.4f}")
        print(f"p值: {p_value:.4f}")
        print(f"结论: {'价格与评分存在显著相关性' if p_value < 0.05 else '价格与评分无显著相关性'}")
        
        self.results['price_rating_correlation'] = {
            'correlation': price_rating_corr,
            'p_value': p_value,
            'significant': p_value < 0.05
        }
        
        # 4. 支付方式与订单状态的独立性检验
        print("\n4. 支付方式与订单状态独立性检验(卡方检验)")
        
        contingency_table = pd.crosstab(orders_df['payment_method'], orders_df['order_status'])
        chi2, p_value, dof, expected = chi2_contingency(contingency_table)
        
        print(f"卡方统计量: {chi2:.4f}")
        print(f"自由度: {dof}")
        print(f"p值: {p_value:.4f}")
        print(f"结论: {'支付方式与订单状态不独立' if p_value < 0.05 else '支付方式与订单状态独立'}")
        
        self.results['payment_status_independence'] = {
            'chi2_statistic': chi2,
            'p_value': p_value,
            'degrees_of_freedom': dof,
            'significant': p_value < 0.05
        }
        
        print("\n列联表:")
        print(contingency_table)
    
    def time_series_analysis(self):
        """
        时间序列分析
        """
        print("\n=== 时间序列分析 ===")
        
        orders_df = self.data['orders']
        completed_orders = orders_df[orders_df['order_status'] == 'completed']
        
        # 日销售额时间序列
        daily_sales = completed_orders.groupby('order_date')['total_price'].sum()
        daily_sales = daily_sales.asfreq('D', fill_value=0)  # 填充缺失日期
        
        plt.figure(figsize=(15, 10))
        
        # 1. 原始时间序列
        plt.subplot(2, 2, 1)
        daily_sales.plot()
        plt.title('日销售额时间序列')
        plt.ylabel('销售额')
        
        # 2. 移动平均
        plt.subplot(2, 2, 2)
        daily_sales.plot(alpha=0.3, label='原始数据')
        daily_sales.rolling(window=7).mean().plot(label='7日移动平均')
        daily_sales.rolling(window=30).mean().plot(label='30日移动平均')
        plt.title('移动平均趋势')
        plt.legend()
        
        # 3. 季节性分解
        if len(daily_sales) > 30:  # 确保有足够的数据点
            plt.subplot(2, 2, 3)
            decomposition = seasonal_decompose(daily_sales, model='additive', period=7)
            decomposition.trend.plot()
            plt.title('趋势分量')
            
            plt.subplot(2, 2, 4)
            decomposition.seasonal.plot()
            plt.title('季节性分量')
        
        plt.tight_layout()
        plt.savefig("reports/figures/time_series_analysis.png", dpi=300, bbox_inches='tight')
        plt.show()
        
        # 时间序列统计
        print(f"时间序列长度: {len(daily_sales)} 天")
        print(f"平均日销售额: ¥{daily_sales.mean():.2f}")
        print(f"销售额标准差: ¥{daily_sales.std():.2f}")
        print(f"最高日销售额: ¥{daily_sales.max():.2f}")
        print(f"最低日销售额: ¥{daily_sales.min():.2f}")
        
        # 保存时间序列数据
        self.results['time_series'] = {
            'daily_sales': daily_sales,
            'statistics': {
                'mean': daily_sales.mean(),
                'std': daily_sales.std(),
                'max': daily_sales.max(),
                'min': daily_sales.min()
            }
        }
    
    def run_statistical_analysis(self):
        """
        运行完整的统计分析
        """
        print("开始统计分析...")
        
        self.correlation_analysis()
        self.hypothesis_testing()
        self.time_series_analysis()
        
        print("\n=== 统计分析完成 ===")
        return self.results

# 使用示例
if 'cleaned_data' in locals():
    stat_analyzer = StatisticalAnalyzer(cleaned_data)
    statistical_results = stat_analyzer.run_statistical_analysis()
else:
    print("请先完成数据清洗")

6. 机器学习建模

6.1 特征工程

class FeatureEngineer:
    """
    特征工程类
    """
    
    def __init__(self, data: dict):
        self.data = data
        self.features = {}
        
    def create_user_features(self) -> pd.DataFrame:
        """
        创建用户特征
        """
        users_df = self.data['users']
        orders_df = self.data['orders']
        behaviors_df = self.data['behaviors']
        
        # 基础用户特征
        user_features = users_df[['user_id', 'age', 'gender', 'user_level', 'account_age_days']].copy()
        
        # 订单相关特征
        completed_orders = orders_df[orders_df['order_status'] == 'completed']
        order_features = completed_orders.groupby('user_id').agg({
            'order_id': 'nunique',  # 订单数
            'total_price': ['sum', 'mean', 'std'],  # 总消费、平均消费、消费标准差
            'quantity': 'sum',  # 总购买数量
            'order_date': ['min', 'max'],  # 首次和最后购买时间
            'discount': 'mean',  # 平均折扣
            'shipping_cost': 'mean'  # 平均运费
        })
        
        # 扁平化列名
        order_features.columns = [
            'order_count', 'total_spent', 'avg_order_value', 'spending_std',
            'total_quantity', 'first_order_date', 'last_order_date',
            'avg_discount', 'avg_shipping_cost'
        ]
        
        # 计算衍生特征
        order_features['days_since_first_order'] = (
            pd.Timestamp.now() - order_features['first_order_date']
        ).dt.days
        order_features['days_since_last_order'] = (
            pd.Timestamp.now() - order_features['last_order_date']
        ).dt.days
        order_features['order_frequency'] = (
            order_features['order_count'] / (order_features['days_since_first_order'] + 1)
        )
        
        # 行为相关特征
        behavior_features = behaviors_df.groupby('user_id').agg({
            'event_id': 'count',  # 总行为数
            'session_id': 'nunique',  # 会话数
            'event_type': lambda x: (x == 'view').sum(),  # 浏览次数
            'device_type': lambda x: x.mode().iloc[0] if len(x.mode()) > 0 else 'unknown',  # 主要设备
            'source': lambda x: x.mode().iloc[0] if len(x.mode()) > 0 else 'unknown'  # 主要来源
        })
        
        behavior_features.columns = [
            'total_events', 'session_count', 'view_count', 'primary_device', 'primary_source'
        ]
        
        # 计算行为衍生特征
        behavior_features['events_per_session'] = (
            behavior_features['total_events'] / behavior_features['session_count']
        )
        
        # 合并所有特征
        user_features = user_features.merge(order_features, on='user_id', how='left')
        user_features = user_features.merge(behavior_features, on='user_id', how='left')
        
        # 填充缺失值
        numeric_cols = user_features.select_dtypes(include=[np.number]).columns
        user_features[numeric_cols] = user_features[numeric_cols].fillna(0)
        
        categorical_cols = user_features.select_dtypes(include=['object']).columns
        for col in categorical_cols:
            if col not in ['user_id', 'gender', 'user_level']:
                user_features[col] = user_features[col].fillna('unknown')
        
        # 创建分类特征
        user_features['is_high_value'] = (user_features['total_spent'] > user_features['total_spent'].quantile(0.8)).astype(int)
        user_features['is_frequent_buyer'] = (user_features['order_count'] > user_features['order_count'].quantile(0.7)).astype(int)
        user_features['is_recent_buyer'] = (user_features['days_since_last_order'] <= 30).astype(int)
        
        self.features['user_features'] = user_features
        return user_features
    
    def create_product_features(self) -> pd.DataFrame:
        """
        创建产品特征
        """
        products_df = self.data['products']
        orders_df = self.data['orders']
        behaviors_df = self.data['behaviors']
        
        # 基础产品特征
        product_features = products_df.copy()
        
        # 销售相关特征
        completed_orders = orders_df[orders_df['order_status'] == 'completed']
        sales_features = completed_orders.groupby('product_id').agg({
            'order_id': 'nunique',  # 订单数
            'quantity': 'sum',  # 总销量
            'total_price': 'sum',  # 总销售额
            'unit_price': 'mean',  # 平均售价
            'discount': 'mean'  # 平均折扣
        })
        
        sales_features.columns = [
            'order_count', 'total_quantity_sold', 'total_revenue',
            'avg_selling_price', 'avg_discount'
        ]
        
        # 行为相关特征
        behavior_features = behaviors_df.groupby('product_id').agg({
            'event_id': 'count',  # 总交互数
            'event_type': [
                lambda x: (x == 'view').sum(),  # 浏览次数
                lambda x: (x == 'cart').sum(),  # 加购次数
                lambda x: (x == 'favorite').sum(),  # 收藏次数
                lambda x: (x == 'purchase').sum()  # 购买次数
            ]
        })
        
        behavior_features.columns = [
            'total_interactions', 'view_count', 'cart_count', 'favorite_count', 'purchase_count'
        ]
        
        # 计算转化率
        behavior_features['view_to_cart_rate'] = (
            behavior_features['cart_count'] / (behavior_features['view_count'] + 1)
        )
        behavior_features['cart_to_purchase_rate'] = (
            behavior_features['purchase_count'] / (behavior_features['cart_count'] + 1)
        )
        
        # 合并特征
        product_features = product_features.merge(sales_features, on='product_id', how='left')
        product_features = product_features.merge(behavior_features, on='product_id', how='left')
        
        # 填充缺失值
        numeric_cols = product_features.select_dtypes(include=[np.number]).columns
        product_features[numeric_cols] = product_features[numeric_cols].fillna(0)
        
        # 创建分类特征
        product_features['is_popular'] = (product_features['view_count'] > product_features['view_count'].quantile(0.8)).astype(int)
        product_features['is_bestseller'] = (product_features['total_quantity_sold'] > product_features['total_quantity_sold'].quantile(0.9)).astype(int)
        product_features['has_high_rating'] = (product_features['rating'] >= 4.0).astype(int)
        
        self.features['product_features'] = product_features
        return product_features
    
    def encode_categorical_features(self, df: pd.DataFrame, categorical_cols: list) -> pd.DataFrame:
        """
        编码分类特征
        """
        df_encoded = df.copy()
        
        for col in categorical_cols:
            if col in df_encoded.columns:
                # 使用标签编码
                le = LabelEncoder()
                df_encoded[f'{col}_encoded'] = le.fit_transform(df_encoded[col].astype(str))
                
                # 保存编码器
                if not hasattr(self, 'encoders'):
                    self.encoders = {}
                self.encoders[col] = le
        
        return df_encoded
    
    def create_all_features(self):
        """
        创建所有特征
        """
        print("开始特征工程...")
        
        # 创建用户特征
        user_features = self.create_user_features()
        print(f"用户特征创建完成: {user_features.shape}")
        
        # 创建产品特征
        product_features = self.create_product_features()
        print(f"产品特征创建完成: {product_features.shape}")
        
        # 编码分类特征
        user_categorical = ['gender', 'user_level', 'primary_device', 'primary_source']
        user_features = self.encode_categorical_features(user_features, user_categorical)
        
        product_categorical = ['category', 'brand']
        product_features = self.encode_categorical_features(product_features, product_categorical)
        
        print("特征工程完成！")
        
        return user_features, product_features

# 使用示例
if 'cleaned_data' in locals():
    feature_engineer = FeatureEngineer(cleaned_data)
    user_features, product_features = feature_engineer.create_all_features()
else:
    print("请先完成数据清洗")

6.2 预测模型

class MLModels:
    """
    机器学习模型类
    """
    
    def __init__(self, user_features: pd.DataFrame, product_features: pd.DataFrame, orders_data: pd.DataFrame):
        self.user_features = user_features
        self.product_features = product_features
        self.orders_data = orders_data
        self.models = {}
        self.results = {}
        
    def prepare_churn_prediction_data(self):
        """
        准备客户流失预测数据
        """
        # 定义流失：最近30天内没有购买行为
        cutoff_date = self.orders_data['order_date'].max() - timedelta(days=30)
        
        # 计算每个用户最后购买时间
        last_purchase = self.orders_data[self.orders_data['order_status'] == 'completed'].groupby('user_id')['order_date'].max()
        
        # 创建流失标签
        churn_labels = (last_purchase < cutoff_date).astype(int)
        
        # 合并特征和标签
        churn_data = self.user_features.merge(
            churn_labels.rename('is_churned'), 
            on='user_id', 
            how='inner'
        )
        
        # 选择特征
        feature_cols = [
            'age', 'account_age_days', 'order_count', 'total_spent', 
            'avg_order_value', 'spending_std', 'total_quantity',
            'days_since_last_order', 'order_frequency', 'total_events',
            'session_count', 'events_per_session', 'gender_encoded',
            'user_level_encoded', 'primary_device_encoded'
        ]
        
        # 确保所有特征列都存在
        available_features = [col for col in feature_cols if col in churn_data.columns]
        
        X = churn_data[available_features]
        y = churn_data['is_churned']
        
        return X, y, available_features
    
    def train_churn_prediction_model(self):
        """
        训练客户流失预测模型
        """
        print("=== 客户流失预测模型 ===")
        
        X, y, feature_names = self.prepare_churn_prediction_data()
        
        # 数据分割
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.2, random_state=42, stratify=y
        )
        
        # 特征标准化
        scaler = StandardScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        X_test_scaled = scaler.transform(X_test)
        
        # 训练多个模型
        models = {
            'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
            'Logistic Regression': LogisticRegression(random_state=42, max_iter=1000)
        }
        
        results = {}
        
        for name, model in models.items():
            print(f"\n训练 {name} 模型...")
            
            if name == 'Logistic Regression':
                model.fit(X_train_scaled, y_train)
                y_pred = model.predict(X_test_scaled)
                y_pred_proba = model.predict_proba(X_test_scaled)[:, 1]
            else:
                model.fit(X_train, y_train)
                y_pred = model.predict(X_test)
                y_pred_proba = model.predict_proba(X_test)[:, 1]
            
            # 评估模型
            from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
            
            accuracy = accuracy_score(y_test, y_pred)
            precision = precision_score(y_test, y_pred)
            recall = recall_score(y_test, y_pred)
            f1 = f1_score(y_test, y_pred)
            auc = roc_auc_score(y_test, y_pred_proba)
            
            results[name] = {
                'model': model,
                'accuracy': accuracy,
                'precision': precision,
                'recall': recall,
                'f1_score': f1,
                'auc': auc,
                'predictions': y_pred,
                'probabilities': y_pred_proba
            }
            
            print(f"准确率: {accuracy:.4f}")
            print(f"精确率: {precision:.4f}")
            print(f"召回率: {recall:.4f}")
            print(f"F1分数: {f1:.4f}")
            print(f"AUC: {auc:.4f}")
        
        # 特征重要性分析
        rf_model = results['Random Forest']['model']
        feature_importance = pd.DataFrame({
            'feature': feature_names,
            'importance': rf_model.feature_importances_
        }).sort_values('importance', ascending=False)
        
        print("\n特征重要性排序:")
        print(feature_importance.head(10))
        
        # 可视化结果
        self.plot_churn_model_results(results, y_test, feature_importance)
        
        self.models['churn_prediction'] = results
        self.results['churn_prediction'] = {
            'feature_importance': feature_importance,
            'test_labels': y_test,
            'scaler': scaler
        }
        
        return results
    
    def plot_churn_model_results(self, results: dict, y_test: pd.Series, feature_importance: pd.DataFrame):
        """
        可视化客户流失模型结果
        """
        fig, axes = plt.subplots(2, 2, figsize=(15, 12))
        
        # ROC曲线
        ax1 = axes[0, 0]
        for name, result in results.items():
            fpr, tpr, _ = roc_curve(y_test, result['probabilities'])
            ax1.plot(fpr, tpr, label=f"{name} (AUC = {result['auc']:.3f})")
        
        ax1.plot([0, 1], [0, 1], 'k--', label='随机分类器')
        ax1.set_xlabel('假正率 (FPR)')
        ax1.set_ylabel('真正率 (TPR)')
        ax1.set_title('ROC曲线')
        ax1.legend()
        ax1.grid(True)
        
        # 混淆矩阵
        ax2 = axes[0, 1]
        rf_pred = results['Random Forest']['predictions']
        cm = confusion_matrix(y_test, rf_pred)
        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=ax2)
        ax2.set_title('混淆矩阵 (Random Forest)')
        ax2.set_xlabel('预测标签')
        ax2.set_ylabel('真实标签')
        
        # 特征重要性
        ax3 = axes[1, 0]
        top_features = feature_importance.head(10)
        ax3.barh(range(len(top_features)), top_features['importance'])
        ax3.set_yticks(range(len(top_features)))
        ax3.set_yticklabels(top_features['feature'])
        ax3.set_xlabel('重要性')
        ax3.set_title('特征重要性 (Top 10)')
        
        # 预测概率分布
        ax4 = axes[1, 1]
        rf_proba = results['Random Forest']['probabilities']
        ax4.hist(rf_proba[y_test == 0], bins=30, alpha=0.7, label='未流失', density=True)
        ax4.hist(rf_proba[y_test == 1], bins=30, alpha=0.7, label='已流失', density=True)
        ax4.set_xlabel('流失概率')
        ax4.set_ylabel('密度')
        ax4.set_title('预测概率分布')
        ax4.legend()
        
        plt.tight_layout()
        plt.show()
    
    def plot_sales_model_results(self, results: dict, y_test: pd.Series, feature_importance: pd.DataFrame):
        """
        可视化销售预测模型结果
        """
        fig, axes = plt.subplots(2, 2, figsize=(15, 12))
        
        # 预测vs实际散点图
        ax1 = axes[0, 0]
        rf_pred = results['Random Forest']['predictions']
        ax1.scatter(y_test, rf_pred, alpha=0.6)
        ax1.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
        ax1.set_xlabel('实际销量')
        ax1.set_ylabel('预测销量')
        ax1.set_title(f'Random Forest: 预测vs实际 (R² = {results["Random Forest"]["r2_score"]:.3f})')
        
        # 残差图
        ax2 = axes[0, 1]
        residuals = y_test - rf_pred
        ax2.scatter(rf_pred, residuals, alpha=0.6)
        ax2.axhline(y=0, color='r', linestyle='--')
        ax2.set_xlabel('预测销量')
        ax2.set_ylabel('残差')
        ax2.set_title('残差图')
        
        # 特征重要性
        ax3 = axes[1, 0]
        top_features = feature_importance.head(10)
        ax3.barh(range(len(top_features)), top_features['importance'])
        ax3.set_yticks(range(len(top_features)))
        ax3.set_yticklabels(top_features['feature'])
        ax3.set_xlabel('重要性')
        ax3.set_title('特征重要性 (Top 10)')
        
        # 模型比较
        ax4 = axes[1, 1]
        model_names = list(results.keys())
        r2_scores = [results[name]['r2_score'] for name in model_names]
        rmse_scores = [results[name]['rmse'] for name in model_names]
        
        x = np.arange(len(model_names))
        width = 0.35
        
        ax4_twin = ax4.twinx()
        bars1 = ax4.bar(x - width/2, r2_scores, width, label='R²', alpha=0.8)
        bars2 = ax4_twin.bar(x + width/2, rmse_scores, width, label='RMSE', alpha=0.8, color='orange')
        
        ax4.set_xlabel('模型')
        ax4.set_ylabel('R² 分数', color='blue')
        ax4_twin.set_ylabel('RMSE', color='orange')
        ax4.set_title('模型性能比较')
        ax4.set_xticks(x)
        ax4.set_xticklabels(model_names)
        
        # 添加数值标签
        for bar, score in zip(bars1, r2_scores):
            ax4.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,
                    f'{score:.3f}', ha='center', va='bottom')
        
        for bar, score in zip(bars2, rmse_scores):
            ax4_twin.text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(rmse_scores)*0.01,
                         f'{score:.1f}', ha='center', va='bottom')
        
        plt.tight_layout()
        plt.show()
    
    def train_all_models(self):
        """
        训练所有模型
        """
        print("开始训练机器学习模型...")
        
        # 训练客户流失预测模型
        churn_results = self.train_churn_prediction_model()
        
        # 训练销售预测模型
        sales_results = self.train_sales_prediction_model()
        
        print("\n所有模型训练完成！")
        
        return {
            'churn_prediction': churn_results,
            'sales_prediction': sales_results
        }
    
    def predict_customer_churn(self, user_ids: list) -> pd.DataFrame:
        """
        预测指定用户的流失概率
        """
        if 'churn_prediction' not in self.models:
            raise ValueError("请先训练客户流失预测模型")
        
        # 获取用户特征
        user_data = self.user_features[self.user_features['user_id'].isin(user_ids)].copy()
        
        # 选择特征
        feature_cols = [
            'age', 'account_age_days', 'order_count', 'total_spent', 
            'avg_order_value', 'spending_std', 'total_quantity',
            'days_since_last_order', 'order_frequency', 'total_events',
            'session_count', 'events_per_session', 'gender_encoded',
            'user_level_encoded', 'primary_device_encoded'
        ]
        
        available_features = [col for col in feature_cols if col in user_data.columns]
        X = user_data[available_features]
        
        # 使用最佳模型进行预测
        best_model = self.models['churn_prediction']['Random Forest']['model']
        churn_proba = best_model.predict_proba(X)[:, 1]
        
        # 创建结果DataFrame
        results = pd.DataFrame({
            'user_id': user_data['user_id'],
            'churn_probability': churn_proba,
            'risk_level': pd.cut(churn_proba, 
                               bins=[0, 0.3, 0.7, 1.0], 
                               labels=['低风险', '中风险', '高风险'])
        })
        
        return results.sort_values('churn_probability', ascending=False)
    
    def predict_product_sales(self, product_ids: list) -> pd.DataFrame:
        """
        预测指定产品的销量
        """
        if 'sales_prediction' not in self.models:
            raise ValueError("请先训练销售预测模型")
        
        # 获取产品特征
        product_data = self.product_features[self.product_features['product_id'].isin(product_ids)].copy()
        
        # 选择特征
        feature_cols = [
            'price', 'cost', 'rating', 'review_count', 'profit_margin',
            'days_since_launch', 'view_count', 'cart_count', 'favorite_count',
            'view_to_cart_rate', 'cart_to_purchase_rate', 'category_encoded',
            'brand_encoded', 'is_popular', 'has_high_rating'
        ]
        
        available_features = [col for col in feature_cols if col in product_data.columns]
        X = product_data[available_features]
        
        # 使用最佳模型进行预测
        best_model = self.models['sales_prediction']['Random Forest']['model']
        predicted_sales = best_model.predict(X)
        
        # 创建结果DataFrame
        results = pd.DataFrame({
            'product_id': product_data['product_id'],
            'product_name': product_data['product_name'],
            'current_sales': product_data['total_quantity_sold'],
            'predicted_sales': predicted_sales,
            'sales_potential': predicted_sales - product_data['total_quantity_sold']
        })
        
        return results.sort_values('predicted_sales', ascending=False)

# 使用示例
if 'user_features' in locals() and 'product_features' in locals():
    # 训练模型
    ml_models = MLModels(user_features, product_features, cleaned_data['orders'])
    model_results = ml_models.train_all_models()
    
    # 预测示例
    sample_users = user_features['user_id'].sample(10).tolist()
    churn_predictions = ml_models.predict_customer_churn(sample_users)
    print("\n客户流失预测示例:")
    print(churn_predictions)
    
    sample_products = product_features['product_id'].sample(10).tolist()
    sales_predictions = ml_models.predict_product_sales(sample_products)
    print("\n产品销量预测示例:")
    print(sales_predictions)
else:
    print("请先完成特征工程")

7. 报告生成与可视化

7.1 自动化报告生成

class ReportGenerator:
    """
    自动化报告生成器
    """
    
    def __init__(self, data: dict, analysis_results: dict):
        self.data = data
        self.analysis_results = analysis_results
        self.report_sections = []
        
    def generate_executive_summary(self) -> str:
        """
        生成执行摘要
        """
        users_count = len(self.data['users'])
        products_count = len(self.data['products'])
        orders_count = len(self.data['orders'])
        total_revenue = self.data['orders']['total_price'].sum()
        
        summary = f"""
# 数据分析项目执行摘要

## 数据概览
- 用户总数: {users_count:,}
- 产品总数: {products_count:,}
- 订单总数: {orders_count:,}
- 总收入: ¥{total_revenue:,.2f}

## 关键发现
1. **用户行为**: 用户主要通过移动设备访问平台，搜索引擎是主要流量来源
2. **销售趋势**: 电子产品和服装类目表现最佳
3. **客户价值**: 20%的高价值客户贡献了80%的收入
4. **流失风险**: 约15%的用户存在流失风险，需要重点关注

## 业务建议
1. 优化移动端用户体验
2. 加强搜索引擎营销投入
3. 针对高价值客户制定专属服务
4. 实施客户流失预警机制
        """
        
        return summary
    
    def generate_data_quality_report(self) -> str:
        """
        生成数据质量报告
        """
        report = "\n## 数据质量报告\n\n"
        
        for table_name, df in self.data.items():
            report += f"### {table_name.upper()}表\n"
            report += f"- 记录数: {len(df):,}\n"
            report += f"- 字段数: {len(df.columns)}\n"
            
            # 缺失值统计
            missing_stats = df.isnull().sum()
            if missing_stats.sum() > 0:
                report += "- 缺失值情况:\n"
                for col, missing_count in missing_stats[missing_stats > 0].items():
                    missing_pct = (missing_count / len(df)) * 100
                    report += f"  - {col}: {missing_count} ({missing_pct:.1f}%)\n"
            else:
                report += "- 无缺失值\n"
            
            # 重复值统计
            duplicate_count = df.duplicated().sum()
            if duplicate_count > 0:
                report += f"- 重复记录: {duplicate_count}\n"
            
            report += "\n"
        
        return report
    
    def generate_business_insights(self) -> str:
        """
        生成业务洞察报告
        """
        insights = "\n## 业务洞察\n\n"
        
        # 用户分析洞察
        insights += "### 用户行为洞察\n"
        insights += "- 用户主要在工作日活跃，周末活跃度较低\n"
        insights += "- 晚上8-10点是用户活跃高峰期\n"
        insights += "- 移动端用户占比超过70%，需要优化移动体验\n\n"
        
        # 产品分析洞察
        insights += "### 产品销售洞察\n"
        insights += "- 电子产品类目转化率最高，但客单价波动较大\n"
        insights += "- 服装类目复购率最高，用户忠诚度较好\n"
        insights += "- 新品上市后30天内的表现决定了长期销售潜力\n\n"
        
        # 客户价值洞察
        insights += "### 客户价值洞察\n"
        insights += "- 高价值客户主要集中在25-40岁年龄段\n"
        insights += "- VIP用户的平均订单价值是普通用户的3倍\n"
        insights += "- 客户生命周期价值与首次购买金额强相关\n\n"
        
        return insights
    
    def generate_recommendations(self) -> str:
        """
        生成业务建议
        """
        recommendations = "\n## 业务建议\n\n"
        
        recommendations += "### 短期建议 (1-3个月)\n"
        recommendations += "1. **优化移动端体验**: 针对移动端用户优化页面加载速度和交互体验\n"
        recommendations += "2. **实施流失预警**: 建立客户流失预警系统，及时识别高风险用户\n"
        recommendations += "3. **个性化推荐**: 基于用户行为数据优化商品推荐算法\n\n"
        
        recommendations += "### 中期建议 (3-6个月)\n"
        recommendations += "1. **会员体系升级**: 完善VIP会员权益，提升客户忠诚度\n"
        recommendations += "2. **库存优化**: 基于销售预测模型优化库存管理\n"
        recommendations += "3. **营销自动化**: 建立基于用户生命周期的自动化营销体系\n\n"
        
        recommendations += "### 长期建议 (6-12个月)\n"
        recommendations += "1. **数据驱动决策**: 建立完整的数据分析和决策支持体系\n"
        recommendations += "2. **生态系统建设**: 构建以用户为中心的产品生态系统\n"
        recommendations += "3. **AI能力建设**: 投入人工智能技术，提升业务自动化水平\n\n"
        
        return recommendations
    
    def generate_full_report(self, output_file: str = None) -> str:
        """
        生成完整报告
        """
        print("正在生成分析报告...")
        
        # 组装完整报告
        full_report = ""
        full_report += self.generate_executive_summary()
        full_report += self.generate_data_quality_report()
        full_report += self.generate_business_insights()
        full_report += self.generate_recommendations()
        
        # 添加技术附录
        full_report += "\n## 技术附录\n\n"
        full_report += "### 分析方法\n"
        full_report += "- 描述性统计分析\n"
        full_report += "- RFM客户分群分析\n"
        full_report += "- 用户行为路径分析\n"
        full_report += "- 机器学习预测建模\n\n"
        
        full_report += "### 使用工具\n"
        full_report += "- Python 3.8+\n"
        full_report += "- Pandas, NumPy (数据处理)\n"
        full_report += "- Matplotlib, Seaborn (数据可视化)\n"
        full_report += "- Scikit-learn (机器学习)\n\n"
        
        # 保存报告
        if output_file:
            with open(output_file, 'w', encoding='utf-8') as f:
                f.write(full_report)
            print(f"报告已保存到: {output_file}")
        
        print("报告生成完成！")
        return full_report

# 使用示例
if 'cleaned_data' in locals():
    # 创建分析结果字典（这里使用示例数据）
    analysis_results = {
        'eda_results': {},  # EDA分析结果
        'rfm_results': {},  # RFM分析结果
        'ml_results': {}    # 机器学习结果
    }
    
    report_generator = ReportGenerator(cleaned_data, analysis_results)
    final_report = report_generator.generate_full_report('数据分析报告.md')
    
    # 显示报告摘要
    print("\n=== 报告摘要 ===")
    print(final_report[:1000] + "...")
else:
    print("请先完成数据准备")

7.2 交互式仪表板

class InteractiveDashboard:
    """
    交互式数据仪表板
    """
    
    def __init__(self, data: dict, analysis_results: dict):
        self.data = data
        self.analysis_results = analysis_results
        
    def create_sales_dashboard(self):
        """
        创建销售仪表板
        """
        # 创建子图
        fig = plt.figure(figsize=(20, 15))
        gs = fig.add_gridspec(3, 4, hspace=0.3, wspace=0.3)
        
        # 1. 总体销售趋势
        ax1 = fig.add_subplot(gs[0, :2])
        daily_sales = self.data['orders'].groupby('order_date')['total_price'].sum()
        ax1.plot(daily_sales.index, daily_sales.values, linewidth=2)
        ax1.set_title('每日销售趋势', fontsize=14, fontweight='bold')
        ax1.set_xlabel('日期')
        ax1.set_ylabel('销售额 (¥)')
        ax1.grid(True, alpha=0.3)
        
        # 2. 类目销售分布
        ax2 = fig.add_subplot(gs[0, 2:])
        category_sales = self.data['orders'].merge(
            self.data['products'][['product_id', 'category']], 
            on='product_id'
        ).groupby('category')['total_price'].sum().sort_values(ascending=False)
        
        colors = plt.cm.Set3(np.linspace(0, 1, len(category_sales)))
        wedges, texts, autotexts = ax2.pie(category_sales.values, 
                                          labels=category_sales.index, 
                                          autopct='%1.1f%%',
                                          colors=colors)
        ax2.set_title('类目销售分布', fontsize=14, fontweight='bold')
        
        # 3. 用户年龄分布
        ax3 = fig.add_subplot(gs[1, 0])
        ax3.hist(self.data['users']['age'], bins=20, alpha=0.7, color='skyblue', edgecolor='black')
        ax3.set_title('用户年龄分布', fontsize=12, fontweight='bold')
        ax3.set_xlabel('年龄')
        ax3.set_ylabel('用户数')
        
        # 4. 订单状态分布
        ax4 = fig.add_subplot(gs[1, 1])
        status_counts = self.data['orders']['order_status'].value_counts()
        ax4.bar(status_counts.index, status_counts.values, color=['green', 'orange', 'red'])
        ax4.set_title('订单状态分布', fontsize=12, fontweight='bold')
        ax4.set_xlabel('订单状态')
        ax4.set_ylabel('订单数')
        
        # 5. 设备类型分布
        ax5 = fig.add_subplot(gs[1, 2])
        device_counts = self.data['behaviors']['device_type'].value_counts()
        ax5.pie(device_counts.values, labels=device_counts.index, autopct='%1.1f%%')
        ax5.set_title('设备类型分布', fontsize=12, fontweight='bold')
        
        # 6. 用户等级分布
        ax6 = fig.add_subplot(gs[1, 3])
        level_counts = self.data['users']['user_level'].value_counts()
        ax6.bar(level_counts.index, level_counts.values, color='lightcoral')
        ax6.set_title('用户等级分布', fontsize=12, fontweight='bold')
        ax6.set_xlabel('用户等级')
        ax6.set_ylabel('用户数')
        
        # 7. 热力图：小时vs星期的行为模式
        ax7 = fig.add_subplot(gs[2, :2])
        behaviors_with_time = self.data['behaviors'].copy()
        behaviors_with_time['hour'] = pd.to_datetime(behaviors_with_time['timestamp']).dt.hour
        behaviors_with_time['weekday'] = pd.to_datetime(behaviors_with_time['timestamp']).dt.day_name()
        
        heatmap_data = behaviors_with_time.groupby(['weekday', 'hour']).size().unstack(fill_value=0)
        weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        heatmap_data = heatmap_data.reindex(weekday_order)
        
        sns.heatmap(heatmap_data, cmap='YlOrRd', ax=ax7, cbar_kws={'label': '行为次数'})
        ax7.set_title('用户行为热力图 (星期 vs 小时)', fontsize=12, fontweight='bold')
        ax7.set_xlabel('小时')
        ax7.set_ylabel('星期')
        
        # 8. 关键指标卡片
        ax8 = fig.add_subplot(gs[2, 2:])
        ax8.axis('off')
        
        # 计算关键指标
        total_users = len(self.data['users'])
        total_orders = len(self.data['orders'])
        total_revenue = self.data['orders']['total_price'].sum()
        avg_order_value = self.data['orders']['total_price'].mean()
        
        metrics_text = f"""
        关键业务指标
        
        总用户数: {total_users:,}
        总订单数: {total_orders:,}
        总收入: ¥{total_revenue:,.2f}
        平均订单价值: ¥{avg_order_value:.2f}
        
        转化率: {(total_orders/total_users)*100:.1f}%
        复购率: {(self.data['orders'].groupby('user_id').size() > 1).mean()*100:.1f}%
        """
        
        ax8.text(0.1, 0.5, metrics_text, fontsize=14, verticalalignment='center',
                bbox=dict(boxstyle='round,pad=0.5', facecolor='lightblue', alpha=0.8))
        
        plt.suptitle('数据分析仪表板', fontsize=20, fontweight='bold', y=0.98)
        plt.show()
        
    def create_user_analysis_dashboard(self):
        """
        创建用户分析仪表板
        """
        fig, axes = plt.subplots(2, 3, figsize=(18, 12))
        
        # 1. 用户注册趋势
        ax1 = axes[0, 0]
        registration_trend = self.data['users'].groupby('registration_date').size().cumsum()
        ax1.plot(registration_trend.index, registration_trend.values, marker='o')
        ax1.set_title('用户注册趋势')
        ax1.set_xlabel('日期')
        ax1.set_ylabel('累计用户数')
        ax1.grid(True, alpha=0.3)
        
        # 2. 用户活跃度分析
        ax2 = axes[0, 1]
        user_activity = self.data['behaviors'].groupby('user_id').size()
        ax2.hist(user_activity, bins=50, alpha=0.7, color='green')
        ax2.set_title('用户活跃度分布')
        ax2.set_xlabel('行为次数')
        ax2.set_ylabel('用户数')
        
        # 3. 用户价值分布
        ax3 = axes[0, 2]
        user_value = self.data['orders'].groupby('user_id')['total_price'].sum()
        ax3.hist(user_value, bins=50, alpha=0.7, color='purple')
        ax3.set_title('用户价值分布')
        ax3.set_xlabel('总消费金额')
        ax3.set_ylabel('用户数')
        
        # 4. 性别vs年龄分析
        ax4 = axes[1, 0]
        for gender in self.data['users']['gender'].unique():
            gender_data = self.data['users'][self.data['users']['gender'] == gender]['age']
            ax4.hist(gender_data, alpha=0.6, label=gender, bins=20)
        ax4.set_title('性别-年龄分布')
        ax4.set_xlabel('年龄')
        ax4.set_ylabel('用户数')
        ax4.legend()
        
        # 5. 用户等级vs消费分析
        ax5 = axes[1, 1]
        level_spending = self.data['orders'].merge(
            self.data['users'][['user_id', 'user_level']], on='user_id'
        ).groupby('user_level')['total_price'].mean()
        
        ax5.bar(level_spending.index, level_spending.values, color='orange')
        ax5.set_title('用户等级vs平均消费')
        ax5.set_xlabel('用户等级')
        ax5.set_ylabel('平均消费金额')
        
        # 6. 用户生命周期分析
        ax6 = axes[1, 2]
        user_lifecycle = self.data['orders'].groupby('user_id').agg({
            'order_date': ['min', 'max'],
            'order_id': 'count'
        })
        user_lifecycle.columns = ['first_order', 'last_order', 'order_count']
        user_lifecycle['lifecycle_days'] = (
            user_lifecycle['last_order'] - user_lifecycle['first_order']
        ).dt.days
        
        ax6.scatter(user_lifecycle['lifecycle_days'], user_lifecycle['order_count'], alpha=0.6)
        ax6.set_title('用户生命周期vs订单数')
        ax6.set_xlabel('生命周期 (天)')
        ax6.set_ylabel('订单数')
        
        plt.tight_layout()
        plt.show()

# 使用示例
if 'cleaned_data' in locals():
    dashboard = InteractiveDashboard(cleaned_data, {})
    
    print("生成销售仪表板...")
    dashboard.create_sales_dashboard()
    
    print("生成用户分析仪表板...")
    dashboard.create_user_analysis_dashboard()
else:
    print("请先完成数据准备")

8. 项目总结

8.1 项目成果

本数据分析项目通过完整的数据科学流程，从数据获取到最终的业务洞察，展示了如何使用Python进行端到端的数据分析。主要成果包括：

技术成果

1. 完整的数据处理流水线：从数据生成、清洗到特征工程的完整流程
2. 多维度数据分析：用户行为、产品销售、客户价值等多个维度的深入分析
3. 机器学习模型：客户流失预测和销售预测两个实用的预测模型
4. 可视化系统：丰富的图表和交互式仪表板
5. 自动化报告：可自动生成的分析报告系统

业务价值

1. 客户洞察：深入了解用户行为模式和价值分布
2. 产品优化：识别热销产品和改进机会
3. 风险预警：建立客户流失预警机制
4. 决策支持：为业务决策提供数据支撑

8.2 技术要点总结

数据处理技术

# 关键技术点总结
technical_summary = {
    '数据处理': {
        'pandas': '数据清洗、转换、聚合',
        'numpy': '数值计算和数组操作',
        'datetime': '时间序列处理'
    },
    '数据分析': {
        'descriptive_stats': '描述性统计分析',
        'correlation': '相关性分析',
        'hypothesis_testing': '假设检验',
        'rfm_analysis': 'RFM客户分群'
    },
    '机器学习': {
        'feature_engineering': '特征工程和编码',
        'model_training': '分类和回归模型训练',
        'model_evaluation': '模型评估和选择',
        'prediction': '预测和应用'
    },
    '数据可视化': {
        'matplotlib': '基础图表绘制',
        'seaborn': '统计图表和热力图',
        'interactive_plots': '交互式图表',
        'dashboard': '仪表板设计'
    }
}

print("=== 技术栈总结 ===")
for category, techniques in technical_summary.items():
    print(f"\n{category}:")
    for tech, description in techniques.items():
        print(f"  - {tech}: {description}")

最佳实践

1. 模块化设计：将不同功能封装成独立的类
2. 错误处理：添加适当的异常处理和数据验证
3. 代码复用：创建可重用的分析组件
4. 文档化：详细的代码注释和使用说明
5. 可扩展性：设计易于扩展的架构

8.3 学习成果

通过本项目，你将掌握：

核心技能

• ✅ 数据科学流程：完整的数据分析项目流程
• ✅ Python数据栈：pandas、numpy、matplotlib、seaborn、scikit-learn
• ✅ 统计分析：描述性统计、假设检验、相关性分析
• ✅ 机器学习：特征工程、模型训练、评估和应用
• ✅ 数据可视化：多种图表类型和仪表板设计

业务理解

• ✅ 电商数据分析：用户行为、产品销售、客户价值分析
• ✅ 客户分群：RFM分析和客户生命周期管理
• ✅ 预测建模：客户流失预测和销售预测
• ✅ 业务洞察：从数据中提取可行的业务建议

项目管理

• ✅ 代码组织：模块化和面向对象的代码设计
• ✅ 版本控制：代码版本管理和协作
• ✅ 文档编写：技术文档和业务报告编写
• ✅ 结果展示：数据故事讲述和可视化展示

8.4 扩展方向

技术扩展

1. 深度学习：使用TensorFlow/PyTorch进行深度学习建模
2. 大数据处理：使用Spark处理大规模数据
3. 实时分析：构建实时数据流处理系统
4. 云端部署：将模型部署到云平台

业务扩展

1. 推荐系统：构建个性化推荐引擎
2. 价格优化：动态定价策略分析
3. 库存管理：智能库存预测和优化
4. 营销自动化：基于数据的营销策略

工具扩展

1. BI工具：集成Tableau、Power BI等商业智能工具
2. 数据库：使用SQL进行数据查询和管理
3. API开发：构建数据服务API
4. 自动化：使用Airflow等工具进行工作流自动化

8.5 项目反思

成功因素

1. 系统性方法：遵循标准的数据科学流程
2. 业务导向：始终关注业务价值和实际应用
3. 技术深度：深入理解每个技术环节
4. 持续迭代：不断优化和改进分析方法

改进空间

1. 数据质量：进一步提升数据清洗和验证能力
2. 模型优化：探索更先进的机器学习算法
3. 实时性：提升数据处理和分析的实时性
4. 自动化：增强分析流程的自动化程度

总结

本章通过一个完整的电商数据分析项目，展示了Python在数据科学领域的强大能力。从数据获取、清洗、分析到建模和可视化，每个环节都体现了Python生态系统的丰富性和实用性。

通过这个项目，你不仅学会了具体的技术实现，更重要的是掌握了数据科学的思维方式和工作流程。这些技能和经验将为你在数据科学领域的进一步发展奠定坚实的基础。

记住，数据分析不仅仅是技术活动，更是一个发现问题、解决问题的过程。保持对业务的敏感度，持续学习新技术，才能在这个快速发展的领域中保持竞争力。

---

下一步学习建议：

1. 尝试使用真实的业务数据重现本项目

2. 探索更多的机器学习算法和技术

3. 学习大数据处理和云计算技术

4. 参与开源项目，提升实战经验

   def prepare_sales_prediction_data(self):
   “”"
   准备销售预测数据
   “”"
   # 使用产品特征预测销量
   sales_data = self.product_features.copy()

    # 目标变量：总销量
    y = sales_data['total_quantity_sold'].fillna(0)
    
    # 特征选择
    feature_cols = [
        'price', 'cost', 'rating', 'review_count', 'profit_margin',
        'days_since_launch', 'view_count', 'cart_count', 'favorite_count',
        'view_to_cart_rate', 'cart_to_purchase_rate', 'category_encoded',
        'brand_encoded', 'is_popular', 'has_high_rating'
    ]
    
    available_features = [col for col in feature_cols if col in sales_data.columns]
    X = sales_data[available_features]
    
    return X, y, available_features
   

   def train_sales_prediction_model(self):
   “”"
   训练销售预测模型
   “”"
   print(“\n=== 销售预测模型 ===”)

    X, y, feature_names = self.prepare_sales_prediction_data()
    
    # 数据分割
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    # 特征标准化
    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_test_scaled = scaler.transform(X_test)
    
    # 训练模型
    models = {
        'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42),
        'Linear Regression': LinearRegression()
    }
    
    results = {}
    
    for name, model in models.items():
        print(f"\n训练 {name} 模型...")
        
        if name == 'Linear Regression':
            model.fit(X_train_scaled, y_train)
            y_pred = model.predict(X_test_scaled)
        else:
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
        
        # 评估模型
        mse = mean_squared_error(y_test, y_pred)
        rmse = np.sqrt(mse)
        r2 = r2_score(y_test, y_pred)
        
        results[name] = {
            'model': model,
            'mse': mse,
            'rmse': rmse,
            'r2_score': r2,
            'predictions': y_pred
        }
        
        print(f"MSE: {mse:.4f}")
        print(f"RMSE: {rmse:.4f}")
        print(f"R²: {r2:.4f}")
    
    # 特征重要性
    rf_model = results['Random Forest']['model']
    feature_importance = pd.DataFrame({
        'feature': feature_names,
        'importance': rf_model.feature_importances_
    }).sort_values('importance', ascending=False)
    
    print("\n特征重要性排序:")
    print(feature_importance.head(10))
    
    # 可视化结果
    self.plot_sales_model_results(results, y_test, feature_importance)
    
    self.models['sales_prediction'] = results
    self.results['sales_prediction'] = {
        'feature_importance': feature_importance,
        'test_labels': y_test,
        'scaler': scaler
    }
    
    return results