2014-10-28
在https://github.com/yusugomori/DeepLearning包含了多种编程语言版本（C、C++、Python、Java、scala）的**逻辑斯谛回归（或者“逻辑回归”， Logistic Regression）**的实现。
我对python版本的代码略做整理和注释，最终内容如下：
# !/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Logistic Regression
References :
- Jason Rennie: Logistic Regression,
http://qwone.com/~jason/writing/lr.pdf
- DeepLearningTutorials
https://github.com/lisa-lab/DeepLearningTutorials
'''
'''
该代码原始版本在：https://github.com/yusugomori/DeepLearning
'''
import sys
import numpy
numpy.seterr(all='ignore')
def sigmoid(x):
return 1. / (1 + numpy.exp(-x))
def softmax(x):
e = numpy.exp(x - numpy.max(x)) # prevent overflow；numpy.max(x)取矩阵中的最大值，结果是一个数字
if e.ndim == 1: # 一维空间，即e为向量
return e / numpy.sum(e, axis=0)
else: # 二维空间
return e / numpy.array([numpy.sum(e, axis=1)]).T # axis -> 每一行求和
class LogisticRegression(object):
def __init__(self, input, label, n_in, n_out):
self.x = input
self.y = label
self.W = numpy.zeros((n_in, n_out)) # initialize W 0； 权重矩阵初始化为0，n_in->样本特征数，n_out->类别数。
self.b = numpy.zeros(n_out) # initialize bias 0； 偏置项初始化为0，
# self.params = [self.W, self.b]
def train(self, lr=0.1, input=None, L2_reg=0.00):
# L2_reg，是L2正则化下的惩罚系数；正则化，即“惩罚”，数越大，惩罚力度越大，
# http://sobuhu.com/ml/2012/12/29/normalization-regularization.html
if input is not None:
self.x = input
# p_y_given_x = sigmoid(numpy.dot(self.x, self.W) + self.b)
p_y_given_x = softmax(numpy.dot(self.x, self.W) + self.b) # 样本特征构成的矩阵×权重矩阵+偏置项
d_y = self.y - p_y_given_x # delta y
self.W += lr * numpy.dot(self.x.T, d_y) - lr * L2_reg * self.W
self.b += lr * numpy.mean(d_y, axis=0)
# cost = self.negative_log_likelihood()
# return cost
def negative_log_likelihood(self):
# 交叉熵：http://en.wikipedia.org/wiki/Cross_entropy
# sigmoid_activation = sigmoid(numpy.dot(self.x, self.W) + self.b)
sigmoid_activation = softmax(numpy.dot(self.x, self.W) + self.b)
cross_entropy = - numpy.mean(
numpy.sum(self.y * numpy.log(sigmoid_activation) +
(1 - self.y) * numpy.log(1 - sigmoid_activation),
axis=1))
return cross_entropy
def predict(self, x):
# return sigmoid(numpy.dot(x, self.W) + self.b)
return softmax(numpy.dot(x, self.W) + self.b)
def test_lr(learning_rate=0.01, n_epochs=200):
# training data
x = numpy.array([[1,1,1,0,0,0],
[1,0,1,0,0,0],
[1,1,1,0,0,0],
[0,0,1,1,1,0],
[0,0,1,1,0,0],
[0,0,1,1,1,0],
[0,0,0,0,0,1],
[0,0,0,0,0,1]])
y = numpy.array([[1, 0, 0], # 类0
[1, 0, 0], # 类0
[1, 0, 0], # 类0
[0, 1, 0], # 类1
[0, 1, 0], # 类1
[0, 1, 0], # 类1
[0, 0, 1], # 类2
[0, 0, 1]]) # 类2
# construct LogisticRegression
classifier = LogisticRegression(input=x, label=y, n_in=6, n_out=3)
# train
for epoch in xrange(n_epochs):
classifier.train(lr=learning_rate, L2_reg=1.)
# cost = classifier.negative_log_likelihood()
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
learning_rate *= 0.95
# test
# x = numpy.array([[1, 1, 0, 0, 0, 0],
# [0, 0, 0, 1, 1, 0],
# [1, 1, 1, 1, 1, 0]])
# print x.shape
predict = classifier.predict(x)
print predict
print numpy.argmax(predict, axis=1) # 每一行最大数的位置
print classifier.negative_log_likelihood()
if __name__ == "__main__":
test_lr()
运行test_lr()函数的结果如下：
[[ 0.56100127 0.25894207 0.18005667]
[ 0.48722535 0.30494234 0.20783232]
[ 0.56100127 0.25894207 0.18005667]
[ 0.25894207 0.56100127 0.18005667]
[ 0.30494234 0.48722535 0.20783232]
[ 0.25894207 0.56100127 0.18005667]
[ 0.29463485 0.29463485 0.41073029]
[ 0.29463485 0.29463485 0.41073029]]
[0 0 0 1 1 1 2 2]
1.26403157582