toto.cc

Go to the documentation of this file.

// -*- C++ -*-

// VecCompressor.cc
// Copyright (C) 2001 Pascal Vincent
// Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// 
//  1. Redistributions of source code must retain the above copyright
//     notice, this list of conditions and the following disclaimer.
// 
//  2. Redistributions in binary form must reproduce the above copyright
//     notice, this list of conditions and the following disclaimer in the
//     documentation and/or other materials provided with the distribution.
// 
//  3. The name of the authors may not be used to endorse or promote
//     products derived from this software without specific prior written
//     permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
// NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// 
// This file is part of the PLearn library. For more information on the PLearn
// library, go to the PLearn Web site at www.plearn.org


/* *******************************************************      
   * $Id: VecCompressor.cc,v 1.1 2002/10/03 07:35:28 plearner Exp $
   ******************************************************* */

#include "VecCompressor.h"

namespace PLearn <%
using namespace std;

  // *******************
  // ** VecCompressor **

signed char* VecCompressor::compressVec(const Vec& v, signed char* data)
{
  real* vdata = v.data();
  signed char* ptr = data;

  // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
  // If mode is '0' abs(count) indicates the number of zeroes, 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to 'I' mode
  //                a 0 count means insert 127 zeros and stay in zero mode
  // If mode is 'F' abs(count) indicates the number of floats that follow 
  //                a positive sign indicates switch to 'I' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 floats and stay in float mode
  // If mode is 'I' abs(count) indicates the number of small integers that follow 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 small integers and stay in 'I' mode

  int l = v.length();

  int i=0;
  real val = vdata[i];

  signed char mode = 'F';
  if(val==0.)
    mode = '0';
  else if(issmallint(val))
    mode = 'I';
  // else 'F'

  int count = 0;
  int istart = 0;
  float fval = 0.;
  signed char* pfval = (signed char*)&fval;

  *ptr++ = mode;

  while(i<l)
    {
      switch(mode)
        {
        case '0':
          istart = i;
          while(i<l && is0(vdata[i]))
              i++;
          count = i - istart;
          while(count>127)
            {
              *ptr++ = 0;
              count -= 127;
            }
          if(i>=l || issmallint(vdata[i]))
            {
              *ptr++ = (signed char)(-count);
              mode = 'I';
            }
          else
            {
              *ptr++ = (signed char)count;
              mode = 'F';
            }
          break;

        case 'I':
          istart = i;
          while(i<l && isI(vdata[i]))
              i++;
          count = i - istart;
          while(count>127)
            {
              *ptr++ = 0;
              int n = 127;
              while(n--)
                *ptr++ = (signed char)vdata[istart++];
              count -= 127;
            }
          if(i>=l || is0(vdata[i]))
            {
              *ptr++ = (signed char)(-count);
              mode = '0';
            }
          else // next value is a floating point
            {
              *ptr++ = (signed char)count;
              mode = 'F';
            }
          while(count--)
            *ptr++ = (signed char)vdata[istart++];                
          break;

        case 'F':
          istart = i;
          val = vdata[i];
          while(i<l && isF(vdata[i]))
            i++;
          count = i - istart;
          while(count>127)
            {
              *ptr++ = 0;
              int n = 127;
              while(n--)
                {
                  fval = (float)vdata[istart++];
                  *ptr++ = pfval[0];
                  *ptr++ = pfval[1];
                  *ptr++ = pfval[2];
                  *ptr++ = pfval[3];
                }
              count -= 127;
            }
          if(i>=l || is0(vdata[i]))
            {
              *ptr++ = (signed char)(-count);
              mode = '0';
            }
          else
            {
              *ptr++ = (signed char)count;
              mode = 'I';
            }
          while(count--)
            {
              fval = (float)vdata[istart++];
              *ptr++ = pfval[0];
              *ptr++ = pfval[1];
              *ptr++ = pfval[2];
              *ptr++ = pfval[3];
            }
        }
    }
  return ptr;
}

void VecCompressor::uncompressVec(signed char* data, const Vec& v)
{
  // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
  // If mode is '0' abs(count) indicates the number of zeroes, 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to 'I' mode
  //                a 0 count means insert 127 zeros and stay in zero mode
  // If mode is 'F' abs(count) indicates the number of floats that follow 
  //                a positive sign indicates switch to 'I' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 floats and stay in float mode
  // If mode is 'I' abs(count) indicates the number of small integers that follow 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 small integers and stay in 'I' mode

  real* vptr = v.data();
  real* vptrend = vptr+v.length();
  signed char* ptr = data;
  signed char mode = *ptr++;
  float fval = 0.;
  signed char* pfval = (signed char*)&fval;
  signed char count;
  while(vptr!=vptrend)
    {
      count = *ptr++;
      switch(mode)
        {
        case '0':
          if(count<0)
            {
              mode = 'I';              
              count = -count;
            }
          else if(count>0)
            mode = 'F';
          else
            count = 127;

          while(count--)
            *vptr++ = 0.;
          break;

        case 'I':
          if(count<0)
            {
              mode = '0';
              count = -count;
            }
          else if(count>0)
            mode = 'F';
          else 
            count = 127;

          while(count--)
            *vptr++ = real(*ptr++);
          break;
          
        case 'F':
          if(count<0)
            {
              mode = '0';
              count = -count;
            }
          else if(count>0)
            mode = 'I';
          else 
            count = 127;

          while(count--)
            {
              pfval[0] = *ptr++;
              pfval[1] = *ptr++;
              pfval[2] = *ptr++;
              pfval[3] = *ptr++;
              *vptr++ = real(fval);
            }
          break;

        default:
          PLERROR("Problem in VecCompressor::uncompressVec this should not happen!!! (wrong data format?)");
        }
    }
}

void VecCompressor::writeCompressedVec(ostream& out, const Vec& v)
{
  real* vdata = v.data();

  // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
  // If mode is '0' abs(count) indicates the number of zeroes, 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to 'I' mode
  //                a 0 count means insert 127 zeros and stay in zero mode
  // If mode is 'F' abs(count) indicates the number of floats that follow 
  //                a positive sign indicates switch to 'I' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 floats and stay in float mode
  // If mode is 'I' abs(count) indicates the number of small integers that follow 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 small integers and stay in 'I' mode

  int l = v.length();

  int i=0;
  real val = vdata[i];

  signed char mode = 'F';
  if(val==0.)
    mode = '0';
  else if(issmallint(val))
    mode = 'I';
  // else 'F'

  int count = 0;
  int istart = 0;
  float fval = 0.;

  write_sbyte(out,mode);

  while(i<l)
    {
      switch(mode)
        {
        case '0':
          istart = i;
          while(i<l && is0(vdata[i]))
              i++;
          count = i - istart;
          while(count>127)
            {
              write_sbyte(out,0);
              count -= 127;
            }
          if(i>=l || issmallint(vdata[i]))
            {
              write_sbyte(out,-count);
              mode = 'I';
            }
          else
            {
              write_sbyte(out,count);
              mode = 'F';
            }
          break;

        case 'I':
          istart = i;
          while(i<l && isI(vdata[i]))
              i++;
          count = i - istart;
          while(count>127)
            {
              write_sbyte(out,0);
              int n = 127;
              while(n--)
                write_sbyte(out,(signed char)vdata[istart++]);
              count -= 127;
            }
          if(i>=l || is0(vdata[i]))
            {
              write_sbyte(out,-count);
              mode = '0';
            }
          else // next value is a floating point
            {
              write_sbyte(out,count);
              mode = 'F';
            }
          while(count--)
            write_sbyte(out,(signed char)vdata[istart++]);                
          break;

        case 'F':
          istart = i;
          while(i<l && isF(vdata[i]))
            i++;
          count = i - istart;
          while(count>127)
            {
              write_sbyte(out,0);
              int n = 127;
              while(n--)
                {
                  fval = (float)vdata[istart++];
                  out.write((char*)&fval,4);
                }
              count -= 127;
            }
          if(i>=l || is0(vdata[i]))
            {
              write_sbyte(out,-count);
              mode = '0';
            }
          else
            {
              write_sbyte(out,count);
              mode = 'I';
            }
          while(count--)
            {
              fval = (float)vdata[istart++];
              out.write((char*)&fval,4);
            }
        }
    }
}

void VecCompressor::readCompressedVec(istream& in, const Vec& v)
{
  // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
  // If mode is '0' abs(count) indicates the number of zeroes, 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to 'I' mode
  //                a 0 count means insert 127 zeros and stay in zero mode
  // If mode is 'F' abs(count) indicates the number of floats that follow 
  //                a positive sign indicates switch to 'I' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 floats and stay in float mode
  // If mode is 'I' abs(count) indicates the number of small integers that follow 
  //                a positive sign indicates switch to 'F' mode
  //                a negative sign indicates switch to '0' mode
  //                a 0 count means insert 127 small integers and stay in 'I' mode

  real* vptr = v.data();
  real* vptrend = vptr+v.length();
  signed char mode = read_sbyte(in);
  float fval = 0.;
  signed char count;

  while(vptr!=vptrend)
    {
      count = read_sbyte(in);
      // cerr << int(count) <<  ' ';
      switch(mode)
        {
        case '0':
          if(count<0)
            {
              mode = 'I';              
              count = -count;
            }
          else if(count>0)
            mode = 'F';
          else
            count = 127;

          while(count--)
            *vptr++ = 0.;
          break;

        case 'I':
          if(count<0)
            {
              mode = '0';
              count = -count;
            }
          else if(count>0)
            mode = 'F';
          else 
            count = 127;

          while(count--)
            *vptr++ = real(read_sbyte(in));
          break;
          
        case 'F':
          if(count<0)
            {
              mode = '0';
              count = -count;
            }
          else if(count>0)
            mode = 'I';
          else 
            count = 127;

          while(count--)
            {
              in.read((char*)&fval,4);
              *vptr++ = real(fval);
            }
          break;

        default:
          PLERROR("Problem in VecCompressor::readCompressedVec this should not happen!!! (wrong data format?)");
        }
    }
  // cerr << endl;
}


/*

Format description:

A succession of [ mode-byte, optionally followed by specificaitons of length N, followed by data ]

The way N is encoded will be explained later.

The bits of the mode-byte are interpreted as follows:
* Most significant bit: 
0 : insert the following N values of type T
1 : insert N zeroes and then the following single value of type T

* Next 2 bits indicate the data type T
00 : ones (that's just 1.0, no further data is given to provide the value) 
01 : small 1 byte signed integers 
10 : 4 byte float  
11 : 8 byte double

In all but the 00 case, 1 or N corresponding values of type T are to be read in the stream (after possibly reading N)

* Next 5 bits (values between 0 .. 31) indicate how to get the number N, 

0 : N is given in the following 1 byte unsigned char
1..29: N is that particular value (between 1 and 29)
30: N is given in the following 2 byte unsigned short
31: N is given in the following 4 byte unsigned int
 

*/


/*
size_t my_read_compressed(istream& in, double* vec, int l)
{
  size_t nbytes = 0; // number of bytes read
  unsigned char mode; // the mode byte
  unsigned int N = 0; // N (number of 0s or values to insert) 

  while(l)
    {
      in.get(mode); 
      ++nbytes;
      unsigned char N1 = (mode & 07);
      switch(N1)
        {
        case 0:  // N is the 1 byte to follow
          in.get(N1);
          ++nbytes;
          N = N1;
          break;
        case 30: // N is the 2 bytes to follow
          unsigned short N2;
          in.read((char*)&N2,2);
          nbytes += 2;
          N = N2;
          break;
        case 31: // N is the 4 bytes to follow
          in.read((char*)&N,4);
          nbytes += 4;
          break;
        default: // N is N1
          N = N1;
        }

      if(mode & (unsigned char)0x80) // most significant bit is on
        { // insert N zeros
          l -= N;
          while(N--)            
            *vec++ = 0;
          N = 1;
        }

      if(!l)  // vec ends with zeroes, so there's no extra single value to append. We're done!
        break;

      l -= N;
      mode = ((mode & ~0x80) >> 5); // get the 2 bits we're interested in
      switch(mode)
        {
        case 0: // type ones
          {
            while(N--)
              *vec++ = 1;
          }
          break;
        case 1: // type signed char
          {
          signed char val;
          nbytes += N;
          while(N--)
            {
              in.get(val);
              *vec++ = val;
            }
          }
          break;
        case 2: // type float
          {
            float val;
            nbytes += N<<2;
            while(N--)
              {
                in.read((char*)&val,sizeof(float));
                *vec++ = val;
              }
          }
          break;
        case 3: // type double
          {
            nbytes += N<<3;
            in.read((char*)vec, N<<3);
            vec += N;
          }
        } 
    }
}

unsigned char get_compr_data_type(double x, bool force_float)
{
  if(x==1.)
    return 0;
  else if(double(char(x))==x)
    return 1;
  else if(force_float || double(float(x))==x)
    return 2;
  return 3;
}

unsigned char get_compr_data_type(float x)
{
  if(x==1.)
    return 0;
  else if(float(char(x))==x)
    return 1;
  return 2;
}

#include <limits>

size_t my_write_mode_and_size(ostream& out, bool insert_zeroes, unsigned int N, unsigned char data_type)
{
  size_t nbytes = 0; // nbytes written
  unsigned char mode = data_type<<5;
  if(insert_zeroes)
    mode |= (unsigned char)0x80;
  if(N<30)
    {
      mode |= (unsigned char)N;
      out.put(mode);
      nbytes = 1;
    }
  else if(N<=UCHAR_MAX)
    {
      out.put(mode);
      out.put((unsigned char)N);
      nbytes = 2;
    }
  else if(N<=USHRT_MAX)
    {
      mode |= (unsigned char)30;
      out.put(mode);
      unsigned short N2 = (unsigned short)N;
      out.write((char*)&N2,sizeof(unsigned short));
      nbytes = 3;
    }
  else // (N<=UINT_MAX)
    {
      mode |= (unsigned char)31;
      out.put(mode);
      unsigned int N4 = (unsigned int)N;
      out.write((char*)&N4,sizeof(unsigned int));
      nbytes = 5;
    }
  return nbytes;
}

size_t my_write_raw_data_as(ostream& out, double *vec, int l, unsigned char data_type)
{
  size_t nbytes = 0; // nbytes written
  switch(data_type)
    {
    case 1:
      nbytes = l;
      while(l--)
        out.put((char)*vec++);
      break;
    case 2:
      nbytes = l*sizeof(float);
      while(l--)
        {
          float val = float(*vec++);
          out.write((char*)&val,sizeof(float));
        }
      break;      
    case 3:
      nbytes = l*sizeof(double);
      out.write((char*)vec,nbytes);
      break;
    }
  return nbytes;
}

// Warning: this is low-level code written for efficiency
size_t my_write_compressed(ostream& out, double* vec, int l, bool force_float=false)
{
  size_t nbytes = 0; // number of bytes read

  for(;;)
    {
      int nzeroes = 0;
      while(l && *vec==0.)
        {
          ++nzeroes;
          ++vec;
          --l;
        }

      int nvals = 0;
      unsigned char data_type = 0;
      if(l)
        {
          double* ptr = vec;
          data_type = get_compr_data_type(*ptr, force_float);
          ++nvals;
          ++ptr;
          --l;
          while(l && *ptr!=0. && get_compr_data_type(*ptr)==data_type)
            {
              ++nvals;
              ++ptr;
              --l;
            }
        }

      // Now we know nzeroes, nvals, and data_type
      // So let's encode it:

      if(nzeroes) // we have zeroes
        {
          // write the code for zeroes followed by a single value
          nbytes += my_write_mode_and_size(out, true, nzeroes, data_type);
          if(nvals) // write the following single value
            {
              nbytes += my_write_raw_data_as(out, vec, 1, data_type);
              ++vec;
              --nvals;
            }
        }

      if(nvals) // we have some remaining values
        {
          nbytes += my_write_mode_and_size(out, false, nvals, data_type);
          nbytes += my_write_raw_data_as(out, vec, nvals, data_type);
        }      

    } // end of for(;;)
  return nbytes;
}

*/

%> // end of namespcae PLearn

---