GaussianProcessRegressor.h

Go to the documentation of this file.

// -*- C++ -*-

// GaussianProcessRegressor.h
//
// Copyright (C) 2003 Yoshua Bengio
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// 
//  1. Redistributions of source code must retain the above copyright
//     notice, this list of conditions and the following disclaimer.
// 
//  2. Redistributions in binary form must reproduce the above copyright
//     notice, this list of conditions and the following disclaimer in the
//     documentation and/or other materials provided with the distribution.
// 
//  3. The name of the authors may not be used to endorse or promote
//     products derived from this software without specific prior written
//     permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
// NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// 
// This file is part of the PLearn library. For more information on the PLearn
// library, go to the PLearn Web site at www.plearn.org


 

/* *******************************************************      
 * $Id: GaussianProcessRegressor.h,v 1.10 2004/07/21 16:30:55 chrish42 Exp $
 ******************************************************* */


#ifndef GaussianProcessRegressor_INC
#define GaussianProcessRegressor_INC

#include "PConditionalDistribution.h"
#include <plearn/ker/Kernel.h>

namespace PLearn {
using namespace std;

class GaussianProcessRegressor: public PConditionalDistribution
{

  public:
    typedef PConditionalDistribution inherited;  
    // Build options

    PP<Kernel> kernel; // kernel = prior covariance on functions
    int n_outputs; // dimension of the target variables
    Vec noise_sd; // output noise standard deviation, for each output dimension
    string Gram_matrix_normalization; // normalization method to apply to Gram matrix:
    // "none": no normalization
    // "centering_a_dot_product": this is the kernel PCA centering
    //     K_{ij} <-- K_{ij} - mean_i(K_ij) - mean_j(K_ij) + mean_{ij}(K_ij)
    // "centering_a_distance": this is the MDS transformation of squared distances to dot products
    //     K_{ij} <-- -0.5(K_{ij} - mean_i(K_ij) - mean_j(K_ij) + mean_{ij}(K_ij))
    // "divisive": this is the spectral clustering and Laplacian eigenmaps normalization
    //     K_{ij} <-- K_{ij}/sqrt(mean_i(K_ij) mean_j(K_ij))
    //
    int max_nb_evectors; // if -1 compute all eigenvectors, o/w compute only that many principal eigenvectors


    // temporary fields that don't need to be saved = NON-OPTIONS

    Mat alpha; // each row has the coefficients of K(x,x_j) in regression for i-th output
    mutable Vec Kxxi; // has K(x,x_i) for current input x
    mutable real Kxx; // has K(x,x)  for current input x
    Mat K; // non-sparse Gram matrix
    Mat eigenvectors; // principal eigenvectors (in the rows!)
    Vec eigenvalues; // and corresponding eigenvalues
    Vec meanK; // meanK[j]=mean_i(K_{ij})
    real mean_allK;

  public:

    GaussianProcessRegressor();
    virtual ~GaussianProcessRegressor();

    virtual void makeDeepCopyFromShallowCopy(map<const void*, void*>& copies);

    virtual void setInput(const Vec& input);
    virtual void setInput_const(const Vec& input) const; // same thing, but cheating (make believe it is const)

    virtual double log_density(const Vec& x) const;

    virtual Vec expectation() const;

    virtual void expectation(Vec expected_y) const;

    virtual Mat variance() const;
    virtual void variance(Vec diag_variances) const;

  private:
    void build_();
    
  public:
    virtual void build();

    virtual void forget();

    virtual int outputsize() const;

    virtual void train();

    virtual void computeOutput(const Vec& input, Vec& output) const;

    virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 
                                         const Vec& target, Vec& costs) const;
                                
    virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
                                       Vec& output, Vec& costs) const;

    virtual void computeCostsOnly(const Vec& input, const Vec& target, Vec& costs) const;
    
  
    virtual TVec<string> getTestCostNames() const;

    virtual TVec<string> getTrainCostNames() const;

    virtual int nTestCosts() const { return 2; }

    virtual int nTrainCosts() const { return 2; }

    int getTestCostIndex(const string& costname) const;

    int getTrainCostIndex(const string& costname) const;

  protected:
    static void declareOptions(OptionList& ol);

    // covariance = K + sigma^2 I
    // multiply (K+sigma^2 I)^{-1} by vector v, put result in Cinv_v
    // TRICK USING PRINCIPAL E-VECTORS OF K:
    //   Let C = sum_{i=1}^m lambda_i v_i v_i' + sigma I
    //   with v_i orthonormal eigenvectors. Then, it can also be written
    //       C = sum_{i=1}^m (lambda_i +sigma) v_i v_i' + sum_{i=m+1}^n sigma v_i v_i'
    //   whose inverse is simply
    //       inverse(C) = sum_{i=1}^m 1/(lambda_i +sigma) v_i v_i' + sum_{i=m+1}^n 1/sigma v_i v_i'
    //                  = sum_{i=1}^m (1/(lambda_i +sigma) - 1/sigma) v_i v_i' + 1/sigma I
    // set Cinv_v = inverse(C)*v, using given sigma in C
    void inverseCovTimesVec(real sigma, Vec v, Vec Cinv_v) const;
    // return u'*inverse(C)*u, using given sigma in C
    real QFormInverse(real sigma2, Vec u) const;

    real BayesianCost();

  public:
    PLEARN_DECLARE_OBJECT(GaussianProcessRegressor);

};

DECLARE_OBJECT_PTR(GaussianProcessRegressor);

} // end of namespace PLearn

#endif

---