逻辑回归权重更新：从交叉熵到梯度下降的完整推导

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

import numpy as np

class LogisticRegressionGD:
    def __init__(self, lr=0.1, n_iter=1000, fit_intercept=True):
        self.lr = lr              # 学习率 η
        self.n_iter = n_iter      # 迭代轮数
        self.fit_intercept = fit_intercept
        self.w = None             # 权重向量
        self.loss_history = []
    
    def _sigmoid(self, z):
        """Sigmoid: σ(z) = 1 / (1 + exp(-z))"""
        return 1 / (1 + np.exp(-np.clip(z, -500, 500)))
    
    def fit(self, X, y):
        """
        X: (N, d)
        y: (N,)
        """
        if self.fit_intercept:
            # 添加一列 1 作为偏置项
            X = np.c_[np.ones(X.shape[0]), X]
        
        N, d = X.shape
        self.w = np.zeros(d)
        
        for epoch in range(self.n_iter):
            # 前向传播：z = X @ w, p = σ(z)
            z = X @ self.w
            p = self._sigmoid(z)
            
            # 计算交叉熵损失
            loss = -np.mean(
                y * np.log(p + 1e-15) + 
                (1 - y) * np.log(1 - p + 1e-15)
            )
            self.loss_history.append(loss)
            
            # 计算梯度: ∇L = (1/N) * X^T @ (p - y)
            grad = (X.T @ (p - y)) / N
            
            # 权重更新: w = w - η * ∇L
            self.w -= self.lr * grad
        
        return self
    
    def predict_proba(self, X):
        if self.fit_intercept:
            X = np.c_[np.ones(X.shape[0]), X]
        return self._sigmoid(X @ self.w)
    
    def predict(self, X):
        return (self.predict_proba(X) >= 0.5).astype(int)


# ========== 验证 ==========
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# 构造二分类数据
X, y = make_classification(
    n_samples=1000, n_features=5, n_informative=3,
    n_redundant=0, n_classes=2, random_state=42
)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 训练
model = LogisticRegressionGD(lr=0.5, n_iter=2000)
model.fit(X_train, y_train)

# 评估
y_pred = model.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"Final loss: {model.loss_history[-1]:.6f}")

# 查看损失下降曲线
import matplotlib.pyplot as plt

plt.figure(figsize=(8, 4))
plt.plot(model.loss_history)
plt.xlabel('Iteration')
plt.ylabel('Cross-Entropy Loss')
plt.title('Loss Curve during Gradient Descent')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('loss_curve.png', dpi=150)
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

def numerical_gradient(X, y, w, eps=1e-5):
    """用有限差分近似梯度"""
    grad = np.zeros_like(w)
    for j in range(len(w)):
        w_plus = w.copy()
        w_plus[j] += eps
        w_minus = w.copy()
        w_minus[j] -= eps
        
        # 分别在 w+ε 和 w-ε 处计算损失
        p_plus = sigmoid(X @ w_plus)
        loss_plus = -np.mean(y * np.log(p_plus) + (1-y) * np.log(1-p_plus))
        
        p_minus = sigmoid(X @ w_minus)
        loss_minus = -np.mean(y * np.log(p_minus) + (1-y) * np.log(1-p_minus))
        
        grad[j] = (loss_plus - loss_minus) / (2 * eps)
    return grad

# 验证：解析梯度与数值梯度应该非常接近
analytical = (X.T @ (p - y)) / N
numerical = numerical_gradient(X, y, w)
print(f"Max diff: {np.max(np.abs(analytical - numerical)):.8f}")
# 期望输出: Max diff < 1e-6