#include "vislearning/features/localfeatures/LFColorWeijer.h" #include #include #include "vislearning/baselib/ColorSpace.h" using namespace OBJREC; using namespace std; using namespace NICE; //! color representation for visualization const int colors[11][3] = { {0, 0, 0}, // black {0, 0, 255}, // blue {165, 42, 42}, // brown {190, 190, 190}, // grey { 0, 255, 0}, // green {255, 165, 0}, // orange {255, 192, 203}, // pink {160, 32, 240}, // purple {255, 0, 0}, // red {255, 255, 255}, // white {255, 255, 0}, // yellow }; LFColorWeijer::LFColorWeijer( const Config *c ) { conf = c; bin[0] = conf->gI( "LFColorWeijer", "binL", 10 ); bin[1] = conf->gI( "LFColorWeijer", "bina", 20 ); bin[2] = conf->gI( "LFColorWeijer", "binb", 20 ); maxv[0] = 100.0; maxv[1] = 80.0; maxv[2] = 50.0; minv[0] = 0.0; minv[1] = -105.0; minv[2] = -200.0; tfile = conf->gS( "LFColorWeijer", "table", "/home/dbv/bilder/colorWeijer/color.txt" ); for ( int i = 0; i < 3; i++ ) { interval[i] = ( maxv[i] - minv[i] ) / ( double )bin[i]; } ifstream test( tfile.c_str() ); if ( test ) { restore(); } else { train(); } } LFColorWeijer::~LFColorWeijer() { for ( uint i = 0; i < hist.size(); i++ ) { for ( uint j = 0; j < hist[i].size(); j++ ) { hist[i][j].clear(); } hist[i].clear(); } hist.clear(); } int LFColorWeijer::getDescSize() const { return LASTCOLOR; } void LFColorWeijer::store() { ofstream fout( tfile.c_str(), ios_base::app ); fout << hist.size() << " " << hist[0].size() << " " << hist[0][0].size() << " " << hist[0][0][0].size() << endl; for ( uint i = 0; i < hist.size(); i++ ) { for ( uint i0 = 0; i0 < hist[i].size(); i0++ ) { for ( uint i1 = 0; i1 < hist[i][i0].size(); i1++ ) { for ( uint i2 = 0; i2 < hist[i][i0][i1].size(); i2++ ) { fout << hist[i][i0][i1][i2] << " "; } } } } } void LFColorWeijer::smooth() { int size0 = ( int )hist.size(); int size1 = ( int )hist[0].size(); int size2 = ( int )hist[0][0].size(); int size3 = ( int )hist[0][0][0].size(); for ( int i0 = 0; i0 < size1; i0++ ) { for ( int i1 = 0; i1 < size2; i1++ ) { for ( int i2 = 0; i2 < size3; i2++ ) { double maxval = 0.0; for ( int i = 0; i < size0; i++ ) { maxval = std::max( maxval, hist[i][i0][i1][i2] ); } if ( maxval == 0.0 ) { for ( int i = 0; i < size0; i++ ) { int anz = 0; for ( int a0 = std::max( i0 - 1, 0 ); a0 <= std::min( i0 + 1, size1 - 1 ); a0++ ) { for ( int a1 = std::max( i1 - 1, 0 ); a1 <= std::min( i1 + 1, size2 - 1 ); a1++ ) { for ( int a2 = std::max( i2 - 1, 0 ); a2 <= std::min( i2 + 1, size3 - 1 ); a2++ ) { anz++; hist[i][i0][i1][i2] += hist[i][a0][a1][a2]; } } } hist[i][i0][i1][i2] /= anz; } } } } } } void LFColorWeijer::restore() { int size0, size1, size2, size3; ifstream fin( tfile.c_str() ); fin >> size0; fin >> size1; fin >> size2; fin >> size3; hist.clear(); for ( int i = 0; i < size0; i++ ) { vector > > v2; for ( int i0 = 0; i0 < size1; i0++ ) { vector > v1; for ( int i1 = 0; i1 < size2; i1++ ) { vector v0; for ( int i2 = 0; i2 < size3; i2++ ) { double val; fin >> val; v0.push_back( val ); } v1.push_back( v0 ); } v2.push_back( v1 ); } hist.push_back( v2 ); } } int LFColorWeijer::getDescriptors( const NICE::Image & img, VVector & positions, VVector & features ) const { cerr << "this are COLOR Features, they won't work on gray value images" << endl; exit( -1 ); } int LFColorWeijer::getDescriptors( const NICE::ColorImage & img, VVector & positions, VVector & features ) const { // in Lab umwandeln for ( int i = 0; i < ( int )positions.size(); i++ ) { vector vals; vector b; int x = positions[i][0]; int y = positions[i][1]; double R, G, B, X, Y, Z; vector lab( 3, 0.0 ); R = ( double )img.getPixel( x, y, 0 ) / 255.0; G = ( double )img.getPixel( x, y, 1 ) / 255.0; B = ( double )img.getPixel( x, y, 2 ) / 255.0; ColorConversion::ccRGBtoXYZ( R, G, B, &X, &Y, &Z, 0 ); ColorConversion::ccXYZtoCIE_Lab( X, Y, Z, &lab[0], &lab[1], &lab[2], 0 ); for ( int i = 0; i < 3; i++ ) { int val = ( int )(( lab[i] - minv[i] ) / interval[i] ); val = std::min( val, bin[i] - 1 ); val = std::max( val, 0 ); b.push_back( val ); } Vector feat( hist.size() ); for ( uint i = 0; i < hist.size(); i++ ) { feat[i] = hist[i][b[0]][b[1]][b[2]]; } features.push_back( feat ); } return 1; } void LFColorWeijer::visualizeFeatures( NICE::Image & mark, const VVector & positions, size_t color ) const { } void LFColorWeijer::add( vector > > &dest, vector > > &src ) { for ( uint i0 = 0; i0 < src.size(); i0++ ) { for ( uint i1 = 0; i1 < src[i0].size(); i1++ ) { for ( uint i2 = 0; i2 < src[i0][i1].size(); i2++ ) { dest[i0][i1][i2] += src[i0][i1][i2]; } } } } int LFColorWeijer::findColor( string &fn ) { if ( fn.find( "black" ) != string::npos ) return BLACK; if ( fn.find( "blue" ) != string::npos ) return BLUE; if ( fn.find( "brown" ) != string::npos ) return BROWN; if ( fn.find( "grey" ) != string::npos ) return GREY; if ( fn.find( "green" ) != string::npos ) return GREEN; if ( fn.find( "orange" ) != string::npos ) return ORANGE; if ( fn.find( "pink" ) != string::npos ) return PINK; if ( fn.find( "purple" ) != string::npos ) return PURPLE; if ( fn.find( "red" ) != string::npos ) return RED; if ( fn.find( "white" ) != string::npos ) return WHITE; if ( fn.find( "yellow" ) != string::npos ) return YELLOW; return -1; } vector > > LFColorWeijer::createTable() { vector > > h; for ( int i0 = 0; i0 < bin[0]; i0++ ) { vector > vec; for ( int i1 = 0; i1 < bin[1]; i1++ ) { vector v; for ( int i2 = 0; i2 < bin[2]; i2++ ) { v.push_back( 0.0 ); } vec.push_back( v ); } h.push_back( vec ); } return h; } void LFColorWeijer::normalize( vector > > &tab ) { double sum = 0.0; for ( uint i0 = 0; i0 < tab.size(); i0++ ) { for ( uint i1 = 0; i1 < tab[i0].size(); i1++ ) { for ( uint i2 = 0; i2 < tab[i0][i1].size(); i2++ ) { sum += tab[i0][i1][i2]; } } } for ( uint i0 = 0; i0 < tab.size(); i0++ ) { for ( uint i1 = 0; i1 < tab[i0].size(); i1++ ) { for ( uint i2 = 0; i2 < tab[i0][i1].size(); i2++ ) { tab[i0][i1][i2] /= sum; } } } return; } void LFColorWeijer::createHist( const ColorImage &cimg, vector > > &hist, Image &mask ) { // in Lab umwandeln NICE::MultiChannelImageT genimg, imglab; ColorSpace::ColorImagetoMultiChannelImage( cimg, genimg ); ColorSpace::convert( imglab, genimg, ColorSpace::COLORSPACE_LAB, ColorSpace::COLORSPACE_RGB ); for ( int y = 0; y < cimg.height(); y++ ) { for ( int x = 0; x < cimg.width(); x++ ) { if ( mask.getPixel( x, y ) == 0 ) continue; vector b; for ( int i = 0; i < 3; i++ ) { int val = ( int )(( imglab.get( x, y, i ) - minv[i] ) / interval[i] ); val = std::min( val, bin[i] - 1 ); b.push_back( val ); } hist[b[0]][b[1]][b[2]]++; } } } void LFColorWeijer::train() { cout << "train Starts" << endl; for ( int i = 0; i < LASTCOLOR; i++ ) { vector > > h = createTable(); hist.push_back( h ); } string dir = conf->gS( "LFColorWeijer", "table", "/home/dbv/bilder/colorWeijer/ebay/" ); string images = conf->gS( "LFColorWeijer", "table", "test_images.txt" ); string mask = conf->gS( "LFColorWeijer", "table", "mask_images.txt" ); string imagesfn; string maskfn; ifstream finimg(( dir + images ).c_str() ); ifstream finmask(( dir + mask ).c_str() ); cout << dir + images << endl; cout << dir + mask << endl; // lese bilder und masken ein while ( finimg >> imagesfn && finmask >> maskfn ) { Image mimg( dir + maskfn ); cout << dir + maskfn << endl; ColorImage cimg( dir + imagesfn ); int col = findColor( imagesfn ); vector > > tab = createTable(); createHist( cimg, tab, mimg ); // erzeuge Lab Histogramm des Bildes normalize( tab ); add( hist[col], tab ); } finimg.close(); finmask.close(); // normalisiere alle lookuptables for ( uint i = 0; i < hist.size(); i++ ) { normalize( hist[i] ); } smooth(); store(); } void LFColorWeijer::visualizeFeatures( NICE::ColorImage & out, const VVector & features, const VVector & position ) const { for ( int i = 0; i < ( int )position.size(); i++ ) { int maxpos = 0; double maxval = 0.0; for ( int j = 0; j < ( int )features[i].size(); j++ ) { if ( maxval < features[i][j] ) { maxval = features[i][j]; maxpos = j; } } out.setPixel( position[i][0], position[i][1], colors[maxpos][0], colors[maxpos][1], colors[maxpos][2] ); } } void LFColorWeijer::visualizeFeatures( const NICE::ColorImage & cimg ) const { ColorImage out; visualizeFeatures( cimg, out ); } void LFColorWeijer::visualizeFeatures( const NICE::ColorImage & cimg, NICE::ColorImage &out ) const { VVector pos, feats; for ( int y = 0; y < cimg.height(); y++ ) { for ( int x = 0; x < cimg.width(); x++ ) { Vector vec( 2 ); vec[0] = x; vec[1] = y; pos.push_back( vec ); } } getDescriptors( cimg, pos, feats ); //heatmap for a special class /*ofstream fout("out.txt"); int counter = 0; for ( int y = 0; y < cimg.height(); y++ ) { for ( int x = 0; x < cimg.width(); x++, counter++ ) { fout << feats[counter][8] << " "; } } cout << "counter: " << counter << " feats.size(): " << feats.size() << endl; fout.close();*/ out.resize( cimg.width(), cimg.height() ); out.set( 0, 0, 0 ); visualizeFeatures( out, feats, pos ); ColorImage combinedout( cimg.width()*2, cimg.height() ); int width = ( int )cimg.width(); for ( int y = 0; y < ( int )cimg.height(); y++ ) { for ( int x = 0; x < width; x++ ) { combinedout.setPixel( x, y, cimg.getPixel( x, y, 0 ), cimg.getPixel( x, y, 1 ), cimg.getPixel( x, y, 2 ) ); combinedout.setPixel( x + width, y, out.getPixel( x, y, 0 ), out.getPixel( x, y, 1 ), out.getPixel( x, y, 2 ) ); } } showImage( combinedout, "result" ); } void LFColorWeijer::getFeats( const ColorImage &img, MultiChannelImageT &feats ) { int width = ( int )img.width(); int height = ( int )img.height(); feats.reInit( width, height, hist.size()); NICE::MultiChannelImageT genimg, imglab; ColorSpace::ColorImagetoMultiChannelImage( img, genimg ); ColorSpace::convert( imglab, genimg, ColorSpace::COLORSPACE_LAB, ColorSpace::COLORSPACE_RGB ); for ( int y = 0; y < height; y++ ) { for ( int x = 0; x < width; x++ ) { for ( uint i = 0; i < hist.size(); i++ ) { vector b( 3, 0.0 ); for ( int j = 0; j < 3; j++ ) { int val = ( int )(( imglab.get( x, y, j ) - minv[j] ) / interval[j] ); val = std::min( val, bin[j] - 1 ); val = std::max( val, 0 ); b[j] = val; } feats.set( x, y, hist[i][b[0]][b[1]][b[2]], i ); } } } return; }