VMatrix.h

Go to the documentation of this file.

// -*- C++ -*-
// PLearn (A C++ Machine Learning Library)
// Copyright (C) 1998 Pascal Vincent
// Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
// Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// 
//  1. Redistributions of source code must retain the above copyright
//     notice, this list of conditions and the following disclaimer.
// 
//  2. Redistributions in binary form must reproduce the above copyright
//     notice, this list of conditions and the following disclaimer in the
//     documentation and/or other materials provided with the distribution.
// 
//  3. The name of the authors may not be used to endorse or promote
//     products derived from this software without specific prior written
//     permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
// NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// 
// This file is part of the PLearn library. For more information on the PLearn
// library, go to the PLearn Web site at www.plearn.org


/* *******************************************************      
   * $Id: VMatrix.h,v 1.52 2004/07/21 20:33:22 tihocan Exp $
   ******************************************************* */


#ifndef VMatrix_INC
#define VMatrix_INC

#include <cstdlib>
#include <map>
#include <plearn/base/PP.h>
#include <plearn/math/TMat.h>
#include <plearn/var/VarArray.h>
#include <plearn/io/IntVecFile.h>
#include <plearn/math/StatsCollector.h>
#include <plearn/math/TMat_maths_impl.h>
#include "VMField.h"

namespace PLearn {
using namespace std;

class Ker;
class VMat;
class Func;

class VMatrix: public Object
{
  
private:

  typedef Object inherited;
  friend class VMat;

  mutable FILE* lockf_; 

  mutable Vec get_row;

  mutable Vec dotrow_1;
  mutable Vec dotrow_2;

protected:

  int length_;
  int width_;
  time_t mtime_; // time of "last modification" of files containing the data

  // For training/testing data sets we assume each row is composed of 3 parts
  // An input part, a target part, and a weight part
  // These fields give those parts' lengths
  // They are used by method VMat::

  int inputsize_;
  int targetsize_;
  int weightsize_;

  // are write operations tolerated?
  bool writable;

  string metadatadir; 

  // [DEPRECATED] contains a short name that can be used as part of a filename for results associated with this dataset.
  string alias_;

  // New set of statistics fields:
  mutable TVec<StatsCollector> field_stats;  

  // the string mapping for each fields, in both directions
  mutable TVec<map<string,real> > map_sr; 
  mutable TVec<map<real,string> > map_rs; 
  
private: 

  void build_();

public:

  mutable Array<VMField> fieldinfos; // don't use this directly (deprecated...) call getFieldInfos() instead
  Array<VMFieldStat> fieldstats;

  VMatrix();

  VMatrix(int the_length, int the_width);

  // simply calls inherited::build() then build_() 
  virtual void build();

  static void declareOptions(OptionList & ol);

  void init_map_sr() const { if (map_sr.length()==0) { map_sr.resize(width()); map_rs.resize(width()); } }

  // Data-set info
  // Sample parts sizes
  

  inline void defineSizes(int inputsize, int targetsize, int weightsize=0)
  { inputsize_ = inputsize, targetsize_ = targetsize, weightsize_ = weightsize; }

  void copySizesFrom(VMat m);

  bool looksTheSameAs(VMat m);
  
  inline int inputsize() const { return inputsize_; }
  inline int targetsize() const { return targetsize_; }
  inline int weightsize() const { return weightsize_; }
  inline bool hasWeights() const { return weightsize_>0; }

  virtual void getExample(int i, Vec& input, Vec& target, real& weight);

  #define SPECIAL_FORMAT ((real)3.1e36)

  // Field types...
  void setFieldInfos(const Array<VMField>& finfo);
  bool hasFieldInfos() const;
  Array<VMField>& getFieldInfos() const;
  VMField& getFieldInfos(int fieldindex) const { return getFieldInfos()[fieldindex]; }
  void declareField(int fieldindex, const string& fieldname, VMField::FieldType fieldtype=VMField::UnknownType);
  void declareFieldNames(TVec<string> fnames);

  int fieldIndex(const string& fieldname) const;

  //|< If this fails, the given string is assumed to hold the numerical index, and its
  int getFieldIndex(const string& fieldname_or_num) const;

  string fieldName(int fieldindex) const { return getFieldInfos(fieldindex).name; } 
  TVec<string> fieldNames() const; // Returns the vector of field names
  void unduplicateFieldNames(); // add a numeric suffix to duplic. fieldNames (eg: field.1 field.2 etc..)

  VMField::FieldType fieldType(int fieldindex) const { return getFieldInfos(fieldindex).fieldtype; } 
  VMField::FieldType fieldType(const string& fieldname) const { return fieldType(fieldIndex(fieldname)); } 
  const VMFieldStat& fieldStat(int j) const { return fieldstats[j]; } 
  const VMFieldStat& fieldStat(const string& fieldname) const { return fieldStat(fieldIndex(fieldname)); }

  void printFields(ostream& out) const;
  void printFieldInfo(ostream& out, int fieldnum) const;
  void printFieldInfo(ostream& out, const string& fieldname_or_num) const;

  string fieldheader(int elementcharwidth=8);

  // loads/saves from/to the metadatadir/fieldnames file
  void saveFieldInfos() const;
  void loadFieldInfos() const;
  
  // these 3 functions deal with stringmaps, notes, and binning files (all three called special field info files, or 'SFIF')
  // for each field eventually, I (julien) guess all this info should be wrapped (thus saved, and loaded) in the VMField class

  // SFIFs, are by default located in the directory MyDataset.{amat,vmat,etc}.metadata/FieldInfo/ and are named 'fieldname'.{smap,notes,binning,...}. 
  // In all 3 functions, the parameter ext (given **with** the dot) specifies the extension of the special field info file [smap,notes,binning], and col
  // is the column index you refer to.

  // setSFIFFilename : sets the SFIF with extensions 'ext' to some 'string'. if this string is different
  // from the default filename, the string is actually placed in a new file called [dataset].metadata/FieldInfo/fieldname.[ext].lnk
  // if the 'string' is empty, the default SFIF filename is assumed, which is : [MyDataset].metadata/FieldInfo/fieldname.[ext]
  void setSFIFFilename(int col, string ext, string filepath="");
  void setSFIFFilename(string fieldname, string ext, string filepath="");

  // getSFIFFilename :If a '*.vmat' dataset uses fields from another dataset, how can we keep the field info dependency? To resolve 
  // this issue, a file named __default.lnk containing path 'P' can be placed in the FieldInfo directory of the .vmat. Here's how 
  // the function getSFIFFilename search for a file : if the default SFIF file doesn't exist, it will then search for the default filename +'.lnk'. 
  // if the later neither exists, the __default.lnk file is used if present, and if not, then an empty (thus inexistent) file (with
  // SFIF default filename) is assumed. 
  string getSFIFFilename(int col, string ext);
  string getSFIFFilename(string fieldname, string ext);
  
  // isSFIFDirect : tells whether the SFIF filename is the default filename. (if false, means the field uses the SFIF from another dataset)
  bool isSFIFDirect(int col, string ext);
  bool isSFIFDirect(string fieldname, string ext);

  // string mapping stuff

  // save all string mapings (one .smap file for each field)
  void saveAllStringMappings();
  
  // save a single field's string mapping in file 'fname'
  void saveStringMappings(int col,string fname);

  void addStringMapping(int col, string str, real val);

  real addStringMapping(int col, string str);

  void removeAllStringMappings();

  void removeColumnStringMappings(int c);

  void removeStringMapping(int col, string str);

  void setStringMapping(int col, const map<string,real>& zemap);

  void deleteStringMapping(int col);

  void loadStringMapping(int col);

  void loadAllStringMappings();

  void copyStringMappingsFrom(VMat source);

  virtual string getValString(int col, real val) const;

  virtual const map<string,real>& getStringToRealMapping(int col) const;

  virtual const map<real,string>& getRealToStringMapping(int col) const;

  virtual real getStringVal(int col, const string & str) const;

  virtual string getString(int row, int col) const;

  
  virtual void computeStats(); 
  bool hasStats() const { return fieldstats.size()>0; }
  void saveStats(const string& filename) const;
  void loadStats(const string& filename);

  virtual void setMetaDataDir(const string& the_metadatadir);

  bool hasMetaDataDir() const { return metadatadir!=""; }

  string getMetaDataDir() const; 

  void lockMetaDataDir() const;

  void unlockMetaDataDir() const;

  string getAlias() const { return alias_; }
  void setAlias(const string& the_alias) { alias_ = the_alias; }

  TVec<StatsCollector> getStats() const;
  
  StatsCollector& getStats(int fieldnum) const
  { return getStats()[fieldnum]; }

  TVec<RealMapping> getRanges();

  // PP<ConditionalStatsCollector> getConditionalStats(int condfield);

  // default version calls savePMAT
  virtual void save(const string& filename) const;

  virtual void savePMAT(const string& pmatfile) const;
  virtual void saveDMAT(const string& dmatdir) const;
  virtual void saveAMAT(const string& amatfile, bool verbose=true, bool no_header = false) const;

  inline int width() const 
  { 
  #ifdef BOUNDCHECK
    if (!this) 
      PLERROR("VMATRIX::width() This object has pointer this=NULL");
  #endif 
    return width_; 
  }
  inline int length() const 
  {
  #ifdef BOUNDCHECK
    if (!this) 
      PLERROR("VMATRIX::length() This object has pointer this=NULL");
  #endif
    return length_;
 }

  inline bool isWritable() const { return writable; }

  // this function (used with .vmat datasets), is used to return the filename of fieldInfo files (string maps (.smap) and notes (.notes))
  // It recursively navigates through links until it finds a suitable file (.smap or .notes)
  // Idea : a .metadata/FieldInfo can contain one of these files :
  // (the order show here is the one used by the function to searches the file)

  // fieldName.smap.lnk : containing the actual path+target OR another .lnk file
  // fieldName.smap : the target (the actual string map or comment file)
  // __default.lnk : contains another FieldInfo directory to look for target (typically the 

  // ** Note 1: that target is assumed to be an inexistant file in the directory where none of the previous 3 can be found (since the file exists only when non-empty)
  // ** Note 2: source may not be target
  string resolveFieldInfoLink(string target, string source);

  inline time_t getMtime() const { return mtime_; }

  inline void setMtime(time_t t) { mtime_ = t; }

  virtual real get(int i, int j) const; 

  virtual void put(int i, int j, real value); 

  virtual void getSubRow(int i, int j, Vec v) const; 

  virtual void putSubRow(int i, int j, Vec v);
    
  virtual void appendRow(Vec v);

  virtual void flush();

  void putOrAppendRow(int i, Vec v);

  void forcePutRow(int i, Vec v);

  virtual void getRow(int i, Vec v) const; 

  virtual void putRow(int i, Vec v);
  virtual void fill(real value);
  virtual void getMat(int i, int j, Mat m) const; 
  virtual void putMat(int i, int j, Mat m); 

  virtual void getColumn(int i, Vec v) const;

  virtual bool find(const Vec& input, real tolerance, int* i = 0) const;

  virtual Mat toMat() const;

  virtual void compacify();

  virtual void reset_dimensions() {}

  virtual VMat subMat(int i, int j, int l, int w);

  virtual real dot(int i1, int i2, int inputsize) const;

  inline real dot(int i1, int i2) const { return dot(i1,i2,width()); }

  virtual real dot(int i, const Vec& v) const;

  operator Mat() const { return toMat(); }

  virtual void getRow(int i, VarArray& inputs) const;

  void print(ostream& out) const;
  virtual void oldwrite(ostream& out) const;
  virtual void oldread(istream& in);

  PLEARN_DECLARE_ABSTRACT_OBJECT(VMatrix);
  void makeDeepCopyFromShallowCopy(map<const void*, void*>& copies);


  virtual void evaluateKernel(Ker ker, int v1_startcol, int v1_ncols, 
                              const Vec& v2, const Vec& result, int startrow=0, int nrows=-1) const;

  virtual real evaluateKernelSum(Ker ker, int v1_startcol, int v1_ncols, 
                                 const Vec& v2, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;

  virtual real evaluateKernelWeightedTargetSum(Ker ker, int v1_startcol, int v1_ncols, const Vec& v2, 
                                               int t_startcol, int t_ncols, Vec& targetsum, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;
  
   
  virtual TVec< pair<real,int> > evaluateKernelTopN(int N, Ker ker, int v1_startcol, int v1_ncols, 
                                                    const Vec& v2, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;

  virtual TVec< pair<real,int> > evaluateKernelBottomN(int N, Ker ker, int v1_startcol, int v1_ncols, 
                                                       const Vec& v2, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;


  virtual void accumulateXtY(int X_startcol, int X_ncols, int Y_startcol, int Y_ncols, 
                             Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;


  virtual void accumulateXtX(int X_startcol, int X_ncols, 
                             Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;

  virtual void evaluateSumOfFprop(Func f, Vec& output_result, int nsamples=-1);
  virtual void evaluateSumOfFbprop(Func f, Vec& output_result, Vec& output_gradient, int nsamples=-1);
 
  virtual ~VMatrix();
};

DECLARE_OBJECT_PTR(VMatrix);

} // end of namespace PLearn

#endif

---