1.線性回歸原理

回歸的目的就是預測數值型的目標值，回歸方程就是回歸系數和輸入線性組合的方程，求回歸系數的過程就叫回歸。線性回歸是輸入項和系數相乘在相加，非線性回歸可能認為輸出是輸入的相乘

求回歸系數就是求出誤差函數并對會回歸系數求導，并讓其為0，就可以求出回歸系數

import numpy as npimport matplotlib.pyplot as pldef loadDataSet(filename):    numFeat = len(open(filename).readline().split('/t')) - 1    dataMat = []    labelMat = []    fr = open(filename)    for line in fr.readlines():        lineArr = []        curLine = line.str
def lwlr(testPoint, xArr, yArr, k = 1.0):    xMat = np.mat(xArr)    yMat = np.mat(yArr).T    m = np.shape(xMat)[0]    weights = np.mat(np.eye((m)))    for j in range(m):        diffMat = testPoint - xMat[j,:]        weights[j,j] = np.exp(diffMat*diffMat.T/(-2.0*k**2))    xTx = xMat.T * (weights*xMat)    if np.linalg.det(xTx) == 0.0:        print 'this matrix is singular, cannot do inverse'        return    ws = xTx.I * (xMat.T*(weights*yMat))    return testPoint * wsdef lwlrTest(testArr, xArr, yArr, k = 1.0):    m = np.shape(testArr)[0]    yHat = np.zeros(m)    for i in range(m):        yHat[i] = lwlr(testArr[i], xArr, yArr, k)    return yHat
嶺回歸，當特征的數目比數據的數還多時，我們可以引入嶺回歸來解決問題.嶺回歸就是在XtX上加一個常數在乘單位矩陣，使其非奇異可求逆
def ridgeRegress(xMat, yMat, lam = 0.2):    xTx = xMat.T * xMat    denom = xTx + np.eye(np.shape(xMat)[1])*lam    if np.linalg.det(denom) == 0.0:        print 'this matrix is singular, cannot do inverse'        return    ws = denom.I * (xMat.T*yMat)    return wsdef ridgeTest(xArr, yArr):    xMat = np.mat(xArr); yMat = np.mat(yArr).T    yMean = np.mean(yMat,0)    yMat = yMat - yMean    xMean = np.mean(xMat, 0)    xVar = np.var(xMat, 0)    xMat = (xMat - xMean) / xVar    numTest = 30    wMat = np.ones((numTest, np.shape(xMat)[1]))    for i in range(30):        ws = ridgeRegress(xMat, yMat, np.exp(i-10))        wMat[i,:] = ws.T    return wMat 
前向逐步回歸
def rssError(yArr, yHatArr):    return ((yArr - yHatArr)**2).sum()def regularize(xMat):    inMat = xMat.copy()    inMatMean = np.mean(inMat,0)    inMatVar = np.var(inMat,0)    inMat = (inMat - inMatMean)/inMatVar    return inMatdef stageWise(xArr, yArr, eps = 0.01, numIt = 100):    xMat = np.mat(xArr); yMat = np.mat(yArr).T    yMean = np.mean(yMat,0)    yMat = yMat - yMean    xMat = regularize(xMat)    m,n = np.shape(xMat)    retMat = np.ones((numIt,n))    ws = np.ones((n,1))    wsTest = ws.copy()    wsMax = ws.copy()    for i in range(numIt):        print ws.T        lowestError = np.Inf        for j in range(n):            for sign in [-1,1]:                wsTest = ws.copy()                wsTest[j] += eps * sign                yTest = xMat * wsTest                rssE = rssError(yMat.A, yTest.A)                if rssE < lowestError:                    lowestError = rssE                wsMax = wsTest        ws = wsMax.copy()        retMat[i,:] = ws.T    return retMat