PLearn: SDBVMat.h Source File

00001 // -*- C++ -*- 00002 // SDBVMat.cc: Implementation of VMat/SDB Interface 00003 // 00004 // Copyright (C) 2000 Nicolas Chapados 00005 // Copyright (C) 2001 Yoshua Bengio 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00037 #ifndef NGSDBVMAT_H 00038 #define NGSDBVMAT_H 00039 00040 #include <plearn/vmat/VMat.h> 00041 #include <plearn/vmat/RowBufferedVMatrix.h> 00042 #include <plearn/var/Func.h> 00043 #include "SDBWithStats.h" 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00048 00049 //######################### CLASS SDBVMOUTPUTCODER ######################### 00065 enum SDBVMOutputCoding { 00066 SDBVMUnknownCoding = 0, 00067 SDBVMNumeric, 00068 SDBVMOneHot, 00069 SDBVMOneHotMinus1 00070 00071 00072 }; 00073 00074 class SDBVMOutputCoder : public PPointable 00075 { 00076 public: 00077 SDBVMOutputCoder(SDBVMOutputCoding oc = SDBVMNumeric, 00078 real missing_values_mapping = MISSING_VALUE); 00079 virtual ~SDBVMOutputCoder(); 00080 00082 00084 virtual SDBVMOutputCoding getOutputCoding() const; 00085 00088 virtual void setNumClasses(int n_classes); 00089 00092 virtual int getNumClasses() const; 00093 00095 virtual void setMissingValuesMapping(real missing_values_mapping); 00096 00098 virtual real getMissingValuesMapping() const; 00099 00100 00102 00105 virtual int fieldWidth() const; 00106 00109 virtual void setOutput(real output_value, const Vec& output_field) const; 00110 00111 public: 00122 template <class Mapping> 00123 static int getNumClasses(const Mapping& mapping, 00124 real other_values_mapping, 00125 real missing_values_mapping); 00126 00128 static int handleOtherAndMissing(bool all_int, 00129 int candidate_max, 00130 real other_values_mapping, 00131 real missing_values_mapping); 00132 00133 protected: 00134 SDBVMOutputCoding output_coding_; 00135 int num_classes_; 00136 real missing_values_mapping_; 00137 }; 00138 00139 typedef PP<SDBVMOutputCoder> PSDBVMOutputCoder; 00140 00141 00142 //########################### CLASS NGSDBVMFIELD ########################### 00152 class SDBVMField : public PPointable 00153 { 00154 public: 00156 SDBVMField(real missing_values_mapping=MISSING_VALUE, VMField::FieldType field_type=VMField::UnknownType) 00157 : missing_values_mapping_(missing_values_mapping), 00158 field_type_(field_type) {} 00159 00164 virtual void convertField(const SDBWithStats& sdb, 00165 const Row& theRow, 00166 const Vec& outputField) const = 0; 00167 00169 virtual int fieldWidth() const = 0; 00170 00177 VMField::FieldType fieldType() const { return field_type_; } 00178 00179 bool isContinuous() const { return field_type_ == VMField::Continuous; } 00180 bool isDiscrGeneral() const { return field_type_ == VMField::DiscrGeneral; } 00181 bool isDiscrMonotonic() const { return field_type_ == VMField::DiscrMonotonic; } 00182 bool isDiscrFloat() const { return field_type_ == VMField::DiscrFloat; } 00183 bool isDate() const { return field_type_ == VMField::Date; } 00184 00185 virtual SDBVMOutputCoding getOutputCoding() const; 00186 00187 protected: 00189 void convertMissing(const Vec& output) const; 00190 00191 protected: 00192 real missing_values_mapping_; 00193 VMField::FieldType field_type_; 00194 }; 00195 00196 typedef PP<SDBVMField> PSDBVMField; 00197 00198 00201 00202 class SDBVMSource 00203 { 00204 protected: 00205 FieldPtr sdbfieldptr; 00206 PSDBVMField sdbvmfieldptr; 00207 Vec output; 00208 00209 public: 00210 SDBVMSource(FieldPtr the_sdbfieldptr) 00211 :sdbfieldptr(the_sdbfieldptr) {} 00212 00213 SDBVMSource(PSDBVMField the_sdbvmfieldptr) 00214 :sdbvmfieldptr(the_sdbvmfieldptr), 00215 output(1) 00216 { 00217 if(sdbvmfieldptr->fieldWidth()!=1) 00218 PLERROR("Can't make a SDBVMSource from a SDBVMField whose width is other than 1"); 00219 } 00220 00222 FieldValue getValue(const SDBWithStats& sdb, const Row& row) const 00223 { 00224 if(sdbfieldptr) 00225 return *row.bind(sdbfieldptr); 00226 else 00227 { 00228 sdbvmfieldptr->convertField(sdb,row,output); 00229 return FieldValue(output[0]); 00230 } 00231 } 00232 00235 const FieldStat& getFieldStat(const SDBWithStats& sdb, const Row& row) const 00236 { 00237 if(!sdbfieldptr) 00238 PLERROR("works only if the source is a FieldPtr"); 00239 return sdb.getStat(sdbfieldptr.field_index()); 00240 } 00241 }; 00242 00243 00244 //########################### CLASS NGSDBVMATRIX ########################### 00251 class SDBVMatrix : public RowBufferedVMatrix 00252 { 00253 typedef RowBufferedVMatrix inherited; 00254 typedef vector<PSDBVMField> FieldsVector; 00255 00256 public: 00258 SDBVMatrix(const string& dbname, bool detect_missing=false); 00259 00261 00264 void appendField(const string& name, SDBVMField* new_field); 00265 00267 const FieldsVector& getFields() const { 00268 return fields_; 00269 } 00270 00272 virtual void getRow(int i, Vec v) const; 00273 00275 SDBWithStats& sdb() { 00276 return sdb_; 00277 } 00278 const SDBWithStats& sdb() const { 00279 return sdb_; 00280 } 00281 00282 protected: 00283 SDBWithStats sdb_; 00284 FieldsVector fields_; 00285 mutable Row row_; 00286 bool detect_missing_; 00287 }; 00288 00289 00290 00291 //##### VM Fields ########################################################### 00292 00294 class SDBVMFieldSource1 : public SDBVMField 00295 { 00296 typedef SDBVMField inherited; 00297 00298 public: 00301 SDBVMFieldSource1(SDBVMSource source, 00302 real missing_values_mapping=MISSING_VALUE, 00303 VMField::FieldType field_type=VMField::UnknownType) 00304 : inherited(missing_values_mapping,field_type), source_(source) {} 00305 00306 protected: 00307 SDBVMSource source_; 00308 }; 00309 00311 class SDBVMFieldSource2 : public SDBVMField 00312 { 00313 typedef SDBVMField inherited; 00314 00315 public: 00318 SDBVMFieldSource2(SDBVMSource source1, SDBVMSource source2, 00319 real missing_values_mapping=MISSING_VALUE, 00320 VMField::FieldType field_type=VMField::UnknownType) 00321 : inherited(missing_values_mapping,field_type), source1_(source1), source2_(source2) {} 00322 00323 protected: 00324 SDBVMSource source1_; 00325 SDBVMSource source2_; 00326 }; 00327 00328 00329 //##### Simple Numerical Conversions ######################################## 00330 00333 class SDBVMFieldAsIs : public SDBVMFieldSource1 00334 { 00335 typedef SDBVMFieldSource1 inherited; 00336 00337 public: 00338 SDBVMFieldAsIs(SDBVMSource source, real missing_values_mapping=MISSING_VALUE) 00339 : inherited(source,missing_values_mapping,VMField::Continuous) {} 00340 00341 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00342 const Vec& output) const; 00343 00344 virtual int fieldWidth() const; 00345 }; 00346 00347 00349 class SDBVMFieldNormalize : public SDBVMFieldSource1 00350 { 00351 typedef SDBVMFieldSource1 inherited; 00352 00353 public: 00354 SDBVMFieldNormalize(SDBVMSource source, 00355 real missing_values_mapping=MISSING_VALUE) 00356 : inherited(source,missing_values_mapping,VMField::Continuous) {} 00357 00358 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00359 const Vec& output) const; 00360 00361 virtual int fieldWidth() const; 00362 }; 00363 00364 00366 class SDBVMFieldDivSigma : public SDBVMFieldSource1 00367 { 00368 typedef SDBVMFieldSource1 inherited; 00369 00370 public: 00371 SDBVMFieldDivSigma(SDBVMSource source, 00372 real missing_values_mapping=MISSING_VALUE) 00373 : inherited(source,missing_values_mapping,VMField::Continuous) {} 00374 00375 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00376 const Vec& output) const; 00377 00378 virtual int fieldWidth() const; 00379 }; 00380 00381 00383 class SDBVMFieldAffine : public SDBVMFieldSource1 00384 { 00385 typedef SDBVMFieldSource1 inherited; 00386 00387 public: 00388 SDBVMFieldAffine(SDBVMSource source, real a, real b=0.0, 00389 real missing_values_mapping=MISSING_VALUE) 00390 : inherited(source,missing_values_mapping,VMField::Continuous), a_(a), b_(b) {} 00391 00392 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00393 const Vec& output) const; 00394 00395 virtual int fieldWidth() const; 00396 00397 protected: 00398 real a_, b_; 00399 }; 00400 00401 00404 class SDBVMFieldPosAffine : public SDBVMFieldSource1 00405 { 00406 typedef SDBVMFieldSource1 inherited; 00407 00408 public: 00409 SDBVMFieldPosAffine(SDBVMSource source, real a, real b=0.0, 00410 real missing_values_mapping=MISSING_VALUE) 00411 : inherited(source,missing_values_mapping,VMField::Continuous), a_(a), b_(b) {} 00412 00413 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00414 const Vec& output) const; 00415 00416 virtual int fieldWidth() const; 00417 00418 protected: 00419 real a_, b_; 00420 }; 00421 00422 00424 class SDBVMFieldSignedPower : public SDBVMFieldSource1 00425 { 00426 typedef SDBVMFieldSource1 inherited; 00427 00428 public: 00429 SDBVMFieldSignedPower(SDBVMSource source, real a, 00430 real missing_values_mapping=MISSING_VALUE) 00431 : inherited(source,missing_values_mapping,VMField::Continuous), 00432 a_(a) 00433 { 00434 if (a_<=0.0 || a_>=1.0) 00435 PLERROR("Bad range for a (%f), must be in ]0,1[", a_); 00436 } 00437 00438 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00439 const Vec& output) const; 00440 00441 virtual int fieldWidth() const; 00442 00443 protected: 00444 real a_; 00445 }; 00446 00447 00454 class SDBVMFieldFunc1 : public SDBVMFieldSource1 00455 { 00456 typedef SDBVMFieldSource1 inherited; 00457 00458 public: 00459 SDBVMFieldFunc1(SDBVMSource source, Func func, 00460 real missing_values_mapping=MISSING_VALUE) 00461 : inherited(source,missing_values_mapping,VMField::Continuous), func_(func) {} 00462 00463 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00464 const Vec& output) const; 00465 00466 virtual int fieldWidth() const; 00467 00468 protected: 00469 mutable Func func_; 00470 }; 00471 00472 00480 class SDBVMFieldFunc2 : public SDBVMFieldSource2 00481 { 00482 typedef SDBVMFieldSource2 inherited; 00483 00484 public: 00485 SDBVMFieldFunc2(SDBVMSource source1, SDBVMSource source2, Func func, 00486 real missing_values_mapping=MISSING_VALUE) 00487 : inherited(source1, source2,missing_values_mapping,VMField::Continuous), func_(func) {} 00488 00489 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00490 const Vec& output) const; 00491 00492 virtual int fieldWidth() const; 00493 00494 protected: 00495 mutable Func func_; 00496 }; 00497 00498 00499 00500 //##### Date/Day Processing ################################################# 00501 00503 class SDBVMFieldDate : public SDBVMFieldSource1 00504 { 00505 typedef SDBVMFieldSource1 inherited; 00506 00507 public: 00508 SDBVMFieldDate(SDBVMSource source, real missing_values_mapping=MISSING_VALUE) 00509 : inherited(source,missing_values_mapping,VMField::Date) {} 00510 00511 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00512 const Vec& output) const; 00513 00514 virtual int fieldWidth() const; 00515 }; 00516 00521 class SDBVMFieldDay : public SDBVMFieldSource1 00522 { 00523 typedef SDBVMFieldSource1 inherited; 00524 00525 public: 00526 SDBVMFieldDay(SDBVMSource source,real missing_values_mapping=MISSING_VALUE) 00527 : inherited(source,missing_values_mapping,VMField::Continuous) {} 00528 00529 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00530 const Vec& output) const; 00531 00532 virtual int fieldWidth() const; 00533 }; 00534 00535 00536 00537 00539 class SDBVMFieldMonths : public SDBVMFieldSource1 00540 { 00541 typedef SDBVMFieldSource1 inherited; 00542 00543 public: 00544 SDBVMFieldMonths(SDBVMSource source,real missing_values_mapping=MISSING_VALUE) 00545 : inherited(source,missing_values_mapping,VMField::Continuous) {} 00546 00547 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00548 const Vec& output) const; 00549 00550 virtual int fieldWidth() const; 00551 }; 00552 00553 00555 class SDBVMFieldDateDiff : public SDBVMFieldSource2 00556 { 00557 protected: 00558 PDate refdate; 00559 FieldValue date1_threshold_; 00560 FieldValue date2_threshold_; 00561 char unit; 00562 00563 public: 00566 00577 SDBVMFieldDateDiff(SDBVMSource source1, SDBVMSource source2, 00578 char the_unit = 'D', 00579 FieldValue date1_threshold = FieldValue(), 00580 FieldValue date2_threshold = FieldValue()) 00581 : SDBVMFieldSource2(source1,source2,MISSING_VALUE,VMField::Continuous), 00582 refdate(), date1_threshold_(date1_threshold), 00583 date2_threshold_(date2_threshold), unit(the_unit) {} 00584 00586 SDBVMFieldDateDiff(SDBVMSource source1, 00587 PDate the_refdate, 00588 char the_unit = 'D') 00589 : SDBVMFieldSource2(source1,source1,MISSING_VALUE,VMField::Continuous), 00590 refdate(the_refdate), 00591 date1_threshold_(), date2_threshold_(), unit(the_unit) {} 00592 00593 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00594 const Vec& output) const; 00595 00596 virtual int fieldWidth() const; 00597 }; 00598 00599 00600 //##### Discretes and Mappings ############################################## 00601 00604 class SDBVMFieldDiscrete : public SDBVMFieldSource1 00605 { 00606 typedef SDBVMFieldSource1 inherited; 00607 00608 public: 00615 SDBVMFieldDiscrete(SDBVMSource source, 00616 int num_classes = 0, 00617 real missing_values_mapping = MISSING_VALUE, 00618 SDBVMOutputCoding oc = SDBVMNumeric, 00619 VMField::FieldType ft = VMField::DiscrGeneral); 00620 00622 SDBVMFieldDiscrete(SDBVMSource source, 00623 SDBVMOutputCoder* oc, 00624 int num_classes = 0, 00625 real missing_values_mapping = MISSING_VALUE, 00626 VMField::FieldType ft = VMField::DiscrGeneral); 00627 00632 virtual void convertField(const SDBWithStats& sdb, const Row& row, const Vec& output) const; 00633 00634 virtual int fieldWidth() const; 00635 00636 virtual SDBVMOutputCoding getOutputCoding() const; 00637 00642 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const = 0; 00643 00645 int getNumClasses() const { return num_classes_; } 00646 00649 void setNumClasses(int); 00650 00653 void setOutputCoder(SDBVMOutputCoder* oc) { output_coder_ = oc; } 00654 00655 00656 protected: 00657 int num_classes_; 00658 PSDBVMOutputCoder output_coder_; 00659 }; 00660 00661 typedef PP<SDBVMFieldDiscrete> PSDBVMFieldDiscrete; 00662 00664 class SDBVMFieldDateGreater : public SDBVMFieldDiscrete 00665 { 00666 typedef SDBVMFieldDiscrete inherited; 00667 00668 protected: 00669 PDate ref; 00670 00671 public: 00672 SDBVMFieldDateGreater(SDBVMSource source, PDate the_ref) 00673 : inherited(source),ref(the_ref) 00674 { 00675 setNumClasses(2); 00676 } 00677 00678 virtual void convertField(const SDBWithStats& sdb, const Row& row, 00679 const Vec& output) const; 00680 00681 virtual int fieldWidth() const; 00682 00683 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00684 }; 00685 00692 class SDBVMFieldCodeAsIs : public SDBVMFieldDiscrete 00693 { 00694 typedef SDBVMFieldDiscrete inherited; 00695 00696 public: 00697 SDBVMFieldCodeAsIs(SDBVMSource source, 00698 int num_classes = 0, 00699 real missing_values_mapping = MISSING_VALUE, 00700 SDBVMOutputCoding oc = SDBVMNumeric, 00701 VMField::FieldType ft = VMField::DiscrGeneral) 00702 : inherited(source, num_classes, missing_values_mapping, oc, ft) {} 00703 00705 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00706 }; 00707 00708 00719 class SDBVMFieldRemapReals : public SDBVMFieldDiscrete 00720 { 00721 typedef SDBVMFieldDiscrete inherited; 00722 typedef map<real,real> RealMap; 00723 00724 public: 00726 SDBVMFieldRemapReals(SDBVMSource source, 00727 const string& mappings, 00728 real other_values_mapping = MISSING_VALUE, 00729 real missing_values_mapping = MISSING_VALUE, 00730 SDBVMOutputCoding oc = SDBVMNumeric, 00731 VMField::FieldType ft = VMField::DiscrGeneral); 00732 00739 SDBVMFieldRemapReals(SDBVMSource source, 00740 const FieldStat& field_stat, 00741 real other_values_mapping = MISSING_VALUE, 00742 real missing_values_mapping = MISSING_VALUE, 00743 SDBVMOutputCoding oc = SDBVMNumeric, 00744 VMField::FieldType ft = VMField::DiscrGeneral); 00745 00747 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00748 00751 static RealMap getRealMapping(const string& mappings); 00752 00753 protected: 00754 RealMap real_mapping_; 00755 real other_values_mapping_; 00756 }; 00757 00758 00768 class SDBVMFieldRemapStrings : public SDBVMFieldDiscrete 00769 { 00770 typedef SDBVMFieldDiscrete inherited; 00771 typedef map<string,real> StringMap; 00772 00773 public: 00775 SDBVMFieldRemapStrings(SDBVMSource source, 00776 const string& mappings, 00777 real other_values_mapping = MISSING_VALUE, 00778 real missing_values_mapping = MISSING_VALUE, 00779 SDBVMOutputCoding oc = SDBVMNumeric, 00780 VMField::FieldType ft = VMField::DiscrGeneral); 00781 00788 SDBVMFieldRemapStrings(SDBVMSource source, 00789 const FieldStat& field_stat, 00790 real other_values_mapping = MISSING_VALUE, 00791 real missing_values_mapping = MISSING_VALUE, 00792 SDBVMOutputCoding oc = SDBVMNumeric, 00793 VMField::FieldType ft = VMField::DiscrGeneral); 00794 00796 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00797 00800 static StringMap getStringMapping(const string& mappings); 00801 00802 protected: 00803 StringMap string_mapping_; 00804 real other_values_mapping_; 00805 }; 00806 00807 00815 class SDBVMFieldRemapIntervals : public SDBVMFieldDiscrete 00816 { 00817 typedef SDBVMFieldDiscrete inherited; 00818 00819 public: 00820 SDBVMFieldRemapIntervals(SDBVMSource source, 00821 const string& mappings, 00822 real other_values_mapping = MISSING_VALUE, 00823 real missing_values_mapping = MISSING_VALUE, 00824 SDBVMOutputCoding oc = SDBVMNumeric, 00825 VMField::FieldType ft = VMField::DiscrGeneral); 00826 00828 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00829 00835 static void getIntervals(const string& mappings, 00836 bool& all_int, real& max_of_map, 00837 Vec& intervals_x, Vec& intervals_y); 00838 00839 protected: 00840 Vec intervals_x_; 00841 Vec intervals_y_; 00842 real other_values_mapping_; 00843 }; 00844 00845 00846 //##### Multi Discrete ###################################################### 00847 00854 // x1*(5*2) + x2*2 + x3 00857 // \sum_{i=1}^N x_i \product_{j=i+1}^N y_j 00864 typedef Array<PSDBVMFieldDiscrete> FieldArray; 00865 00866 class SDBVMFieldMultiDiscrete : public SDBVMFieldDiscrete 00867 { 00868 typedef SDBVMFieldDiscrete inherited; 00869 00870 public: 00875 SDBVMFieldMultiDiscrete(const FieldArray& fields, 00876 real missing_values_mapping = MISSING_VALUE, 00877 SDBVMOutputCoding oc = SDBVMNumeric, 00878 VMField::FieldType ft = VMField::DiscrGeneral); 00879 00881 const FieldArray& getFields() const { 00882 return fields_; 00883 } 00884 00885 void setFields(const FieldArray& fields); 00887 00888 00890 virtual real getDiscreteValue(const SDBWithStats& sdb, const Row& row) const; 00891 00892 protected: 00893 FieldArray fields_; 00894 Vec field_multipliers_; 00895 00896 }; 00897 00898 00899 //##### ICBC Stuff ########################################################## 00900 00971 class SDBVMFieldICBCTargets : public SDBVMField 00972 { 00973 typedef SDBVMField inherited; 00974 00975 public: 00976 SDBVMFieldICBCTargets(Schema schema, bool use_roadstar, 00977 bool add_claims_sum_column, bool rescale_by_interval, 00978 bool rescale_by_start_date, Mat& start_date_rescaling_values, 00979 const string& targetname="ALL"); 00980 00981 virtual void convertField(const SDBWithStats& sdb, const Row& row, const Vec& output) const; 00982 00983 virtual int fieldWidth() const { 00984 if (targetname_=="ALL") 00985 return 5+int(use_roadstar_)+int(add_claims_sum_column_)+int(rescale_by_interval_); 00986 else if (targetname_=="sum_all_but_BI") 00987 return 1; 00988 else if (targetname_=="ALLcounts") 00989 return 5; 00990 else if (targetname_=="all_counts_but_BI") 00991 return 4; 00992 else if (targetname_=="ALLseverities") 00993 return 10; 00994 else if (targetname_=="all_severities_but_BI") 00995 return 8; 00996 else 00997 return 1+int(use_roadstar_)+int(add_claims_sum_column_)+int(rescale_by_interval_);} 00998 00999 protected: 01000 bool use_roadstar_; 01001 bool add_claims_sum_column_; 01002 bool rescale_by_interval_; 01003 bool rescale_by_start_date_; 01004 Mat start_date_rescaling_values_; 01005 string targetname_; 01006 int reference_start_date_year_; 01007 int reference_start_date_month_; 01008 SDBVMSource start_date_; 01009 SDBVMSource end_date_; 01010 SDBVMSource bodily_injury_incurred_; 01011 SDBVMSource property_damage_incurred_; 01012 SDBVMSource accident_death_incurred_; 01013 SDBVMSource collision_lou_incurred_; 01014 SDBVMSource comprehensive_incurred_; 01015 //SDBVMSource roadstar_incurred_; 01016 SDBVMSource bodily_injury_count_; 01017 SDBVMSource property_damage_count_; 01018 SDBVMSource accident_death_count_; 01019 SDBVMSource collision_lou_count_; 01020 SDBVMSource comprehensive_count_; 01021 01022 SDBVMSource bodily_injury_severity_; 01023 SDBVMSource property_damage_severity_; 01024 SDBVMSource accident_death_severity_; 01025 SDBVMSource collision_lou_severity_; 01026 SDBVMSource comprehensive_severity_; 01027 }; 01028 01029 01030 class SDBVMFieldHasClaim : public SDBVMField 01031 { 01032 typedef SDBVMField inherited; 01033 01034 public: 01035 SDBVMFieldHasClaim(Schema schema): 01036 inherited(0), 01037 bodily_injury_incurred_(schema("bodily_injury_incurred")), 01038 property_damage_incurred_(schema("property_damage_incurred")), 01039 accident_death_incurred_(schema("accident_death_incurred")), 01040 collision_lou_incurred_(schema("collision_lou_incurred")), 01041 comprehensive_incurred_(schema("comprehensive_incurred")), 01042 roadstar_incurred_(schema("roadstar_incurred")) 01043 {} 01044 virtual void convertField(const SDBWithStats& sdb, const Row& row, Vec& output) const; 01045 virtual int fieldWidth() const { return 1;} 01046 01047 protected: 01048 FieldPtr bodily_injury_incurred_; 01049 FieldPtr property_damage_incurred_; 01050 FieldPtr accident_death_incurred_; 01051 FieldPtr collision_lou_incurred_; 01052 FieldPtr comprehensive_incurred_; 01053 FieldPtr roadstar_incurred_; 01054 }; 01055 01056 01057 class SDBVMFieldSumClaims : public SDBVMField 01058 { 01059 typedef SDBVMField inherited; 01060 01061 public: 01062 SDBVMFieldSumClaims(Schema schema): 01063 inherited(0), 01064 bodily_injury_incurred_(schema("bodily_injury_incurred")), 01065 property_damage_incurred_(schema("property_damage_incurred")), 01066 accident_death_incurred_(schema("accident_death_incurred")), 01067 collision_lou_incurred_(schema("collision_lou_incurred")), 01068 comprehensive_incurred_(schema("comprehensive_incurred")), 01069 roadstar_incurred_(schema("roadstar_incurred")) 01070 {} 01071 virtual void convertField(const SDBWithStats& sdb, 01072 const Row& row, Vec& output) const; 01073 virtual int fieldWidth() const { return 1;} 01074 01075 protected: 01076 FieldPtr bodily_injury_incurred_; 01077 FieldPtr property_damage_incurred_; 01078 FieldPtr accident_death_incurred_; 01079 FieldPtr collision_lou_incurred_; 01080 FieldPtr comprehensive_incurred_; 01081 FieldPtr roadstar_incurred_; 01082 }; 01083 01084 class SDBVMFieldICBCClassification : public SDBVMField 01085 { 01086 typedef SDBVMField inherited; 01087 01088 public: 01089 SDBVMFieldICBCClassification(Schema schema, const string& fieldname="", 01090 const string& tmap_file=""); 01091 01092 virtual void convertField(const SDBWithStats& sdb, 01093 const Row& row, const Vec& output) const; 01094 01095 virtual int fieldWidth() const { return 1; } 01096 01097 protected: 01098 SDBVMSource bodily_injury_incurred_; 01099 SDBVMSource property_damage_incurred_; 01100 SDBVMSource accident_death_incurred_; 01101 SDBVMSource collision_lou_incurred_; 01102 SDBVMSource comprehensive_incurred_; 01103 FieldPtr policy_start_date_; 01104 string fieldname_; 01105 int threshold; 01106 }; 01107 01108 01117 int ICBCpartition(const Vec& claims, real threshold); 01118 01119 01120 //##### Local Aliases Namespace ############################################# 01121 01134 namespace SDBFields 01135 { 01136 typedef SDBVMSource FSource; 01137 typedef SDBVMField FField; 01138 typedef SDBVMFieldSource1 FSource1; 01139 typedef SDBVMFieldSource2 FSource2; 01140 typedef SDBVMFieldAsIs FAsIs; 01141 typedef SDBVMFieldNormalize FNormalize; 01142 typedef SDBVMFieldDivSigma FDivSigma; 01143 typedef SDBVMFieldAffine FAffine; 01144 typedef SDBVMFieldPosAffine FPosAffine; 01145 typedef SDBVMFieldSignedPower FSignedPower; 01146 typedef SDBVMFieldFunc1 FFunc1; 01147 typedef SDBVMFieldFunc2 FFunc2; 01148 typedef SDBVMFieldDate FDate; 01149 typedef SDBVMFieldDateDiff FDateDiff; 01150 typedef SDBVMFieldDateGreater FDateGreater; 01151 typedef SDBVMFieldDay FDay; 01152 typedef SDBVMFieldDiscrete FDiscrete; 01153 typedef SDBVMFieldCodeAsIs FCodeAsIs; 01154 typedef SDBVMFieldRemapReals FRemapReals; 01155 typedef SDBVMFieldRemapStrings FRemapStrings; 01156 typedef SDBVMFieldRemapIntervals FRemapIntervals; 01157 typedef SDBVMFieldMultiDiscrete FMultiDiscrete; 01158 typedef SDBVMFieldICBCTargets FICBCTargets; 01159 typedef SDBVMFieldHasClaim FHasClaim; 01160 typedef SDBVMFieldSumClaims FSumClaims; 01161 typedef SDBVMFieldICBCClassification FICBCClassification; 01162 } 01163 01164 } // end of namespace PLearn 01165 01166 #endif 01167