first context version

semseg/SemSegContextTree.cpp

@@ -13,12 +13,13 @@
 
 #include <omp.h>
 
+#define BOUND(x,min,max) (((x)<(min))?(min):((x)>(max)?(max):(x)))
+
 using namespace OBJREC;
 using namespace std;
 using namespace NICE;
 
 
-
 SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md )
     : SemanticSegmentation ( conf, &(md->getClassNames("train")) )
 {
@@ -31,7 +32,15 @@ SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md
 	
 	minFeats = conf->gI(section, "min_feats", 50 );
 	
-	maxDepth = conf->gI(section, "max_depth", 20 );
+	maxDepth = conf->gI(section, "max_depth", 10 );
+	
+	windowSize = conf->gI(section, "window_size", 16);
+	
+	featsPerSplit = conf->gI(section, "feats_per_split", 200);
+	
+	ops.push_back(new Minus());
+	ops.push_back(new MinusAbs());
+	ops.push_back(new Addition());
 	
 	///////////////////////////////////
 	// Train Segmentation Context Trees
@@ -64,8 +73,6 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 	set<vector<int> >selFeats;
 	map<int,int> e;
 	int featcounter = 0;
-	vector<double> maximum(featsize, -numeric_limits< double >::max());
-	vector<double> minimum(featsize, numeric_limits< double >::max());
 	
 	for(int iCounter = 0; iCounter < imgCount; iCounter++)
 	{
@@ -83,8 +90,6 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 		}
 	}
 	
-	//double fraction = (double)maxSamples/(double)featcounter;
-	
 	vector<double> fraction(a.size(),0.0);
 	for(uint i = 0; i < fraction.size(); i++)
 	{
@@ -94,6 +99,7 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 	featcounter = 0;
 	
 	
+	
 	for(int iCounter = 0; iCounter < imgCount; iCounter++)
 	{
 		int xsize = (int)currentfeats[iCounter].size();
@@ -106,10 +112,6 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 				{
 					int cn = labels[iCounter][x][y];
 					double randD = (double)rand()/(double)RAND_MAX;
-					//cout << "class: " << cn << " thres: "<< fraction<<  " rand: " << randD << " scale: " << a[labelmap[cn]] << " newrand: ";
-					//randD *= a[labelmap[cn]];
-					//cout << randD << endl;
-					//getchar();
 					if(randD < fraction[labelmap[cn]])
 					{
 						vector<int> tmp(3,0);
@@ -118,13 +120,7 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 						tmp[2] = y;
 						featcounter++;
 						selFeats.insert(tmp);
-						
-						e[cn] = e[cn]+1;
-						for(int f= 0; f < featsize; f++)
-						{
-							maximum[f] = std::max(maximum[f], feats[iCounter][x][y][f]);
-							minimum[f] = std::min(minimum[f], feats[iCounter][x][y][f]);
-						}
+						e[cn]++;
 					}
 				}
 
@@ -155,33 +151,54 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 		cout << "only " << featcounter << " feats in current node -> it's a leaf" << endl;
 		return;
 	}
-
-	//omp_set_num_threads(2);
+	
+	
+	vector<int> featsubset;
+	for(int i = 0; i < featsPerSplit; i++)
+	{
+		int randI = ((double)rand()/(double)RAND_MAX*(double)allfeatsize);
+		featsubset.push_back(randI);
+	}
+	
 #pragma omp parallel for private(mapit)
-	for(int f = 0; f < featsize; f++)
+	for(int f = 0; f < featsPerSplit; f++)
 	{
+
+		int x1 = featsel[featsubset[f]][0];
+		int y1 = featsel[featsubset[f]][1];
+		int x2 = featsel[featsubset[f]][2];
+		int y2 = featsel[featsubset[f]][3];
+		int chan = featsel[featsubset[f]][4];
+		int o = featsel[featsubset[f]][5];
+
 		double l_bestig = -numeric_limits< double >::max();
 		double l_splitval = -1.0;
-	
 		set<vector<int> >::iterator it;
+		vector<double> vals;
+
 		for ( it=selFeats.begin() ; it != selFeats.end(); it++ )
+		{
+			int xsize = (int)feats[(*it)[0]].size();
+			int ysize = (int)feats[(*it)[0]][0].size();
+			double val1 = feats[(*it)[0]][BOUND((*it)[1]+x1,0,xsize-1)][BOUND((*it)[2]+y1,0,ysize-1)][chan];
+			double val2 = feats[(*it)[0]][BOUND((*it)[1]+x2,0,xsize-1)][BOUND((*it)[2]+y2,0,ysize-1)][chan];
+			vals.push_back(ops[o]->getVal(val1,val2));
+		}
+		
+		int counter = 0;
+		for ( it=selFeats.begin() ; it != selFeats.end(); it++ , counter++)
 		{
 			set<vector<int> >::iterator it2;
-			double val = feats[(*it)[0]][(*it)[1]][(*it)[2]] [f];
-			
-			//cout << "val: " << val << endl;
-			if(val == maximum[f] || val == minimum[f])
-			{
-				continue;
-			}
+			double val = vals[counter];
 			
 			map<int,int> eL, eR;
 			int counterL = 0, counterR = 0;
-			
-			for ( it2=selFeats.begin() ; it2 != selFeats.end(); it2++ )
+			int counter2 = 0;
+			for ( it2=selFeats.begin() ; it2 != selFeats.end(); it2++, counter2++ )
 			{
 				int cn = labels[(*it2)[0]][(*it2)[1]][(*it2)[2]];
-				if(feats[(*it2)[0]][(*it2)[1]][(*it2)[2]][f] < val)
+				//cout << "vals[counter2] " << vals[counter2] << " val: " <<  val << endl;
+				if(vals[counter2] < val)
 				{
 					//left entropie:
 					eL[cn] = eL[cn]+1;
@@ -211,7 +228,7 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 				rightent += p*log2(p);
 			}
 			rightent = -rightent;
-			
+			//cout << "rightent: " << rightent << " leftent: " << leftent << endl;
 			double ig = globent - rightent - leftent;
 			
 			if(ig > l_bestig)
@@ -220,17 +237,21 @@ void SemSegContextTree::getBestSplit(const vector<vector<vector<vector<double> >
 				l_splitval = val;
 			}
 		}
+		
 #pragma omp critical
 {
+		//cout << "globent: " << globent <<  " bestig " << bestig << " splitfeat: " << splitfeat << " splitval: " << splitval << endl;
+		//cout << "globent: " << globent <<  " l_bestig " << l_bestig << " f: " << p << " l_splitval: " << l_splitval << endl;
+		//cout << "p: " << featsubset[f] << endl;
 		if(l_bestig > bestig)
 		{
 			bestig = l_bestig;
-			splitfeat = f;
+			splitfeat = featsubset[f];
 			splitval = l_splitval;
 		}
 }
 	}
-	//cout << "globent: " << globent <<  " bestig " << bestig << " splitfeat: " << splitfeat << " splitval: " << splitval << endl;
+	cout << "globent: " << globent <<  " bestig " << bestig << " splitfeat: " << splitfeat << " splitval: " << splitval << endl;
 }
 
 void SemSegContextTree::train ( const MultiDataset *md )
@@ -287,7 +308,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		//TODO: resize image?!
 		
 		vector<vector<vector<double> > > feats;
-#if 1
+#if 0
 		lfcw->getFeats(img, feats);
 #else
 		feats = vector<vector<vector<double> > >(xsize,vector<vector<double> >(ysize,vector<double>(3,0.0)));
@@ -325,6 +346,37 @@ void SemSegContextTree::train ( const MultiDataset *md )
 	}
 	pb.hide();
 	
+	int opsize = (int)ops.size();
+	int featdim = (int)allfeats[0][0][0].size();
+	allfeatsize = windowSize*windowSize*windowSize*windowSize*featdim*opsize;
+	
+	for(int x1 = -windowSize/2; x1 < windowSize/2+1; x1++)
+	{
+		for(int y1 = -windowSize/2; y1 < windowSize/2+1; y1++)
+		{
+			for(int x2 = -windowSize/2; x2 < windowSize/2+1; x2++)
+			{
+				for(int y2 = -windowSize/2; y2 < windowSize/2+1; y2++)
+				{
+					for(int f = 0; f < featdim; f++)
+					{
+						for(int o = 0; o < opsize; o++)
+						{
+							vector<int> tmp(6,0);
+							tmp[0] = x1;
+							tmp[1] = y1;
+							tmp[2] = x2;
+							tmp[3] = y2;
+							tmp[4] = f;
+							tmp[5] = o;
+							featsel.push_back(tmp);
+						}
+					}
+				}
+			}
+		}
+	}
+	
 	map<int,int>::iterator mapit;
 	int classes = 0;
 	for(mapit = labelcounter.begin(); mapit != labelcounter.end(); mapit++)
@@ -372,6 +424,12 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		int t = (int) tree.size();
 		for(int i = 0; i < t; i++)
 		{
+			printf("tree[%i]: left: %i, right: %i ", i, tree[i].left, tree[i].right);
+			for(int d = 0; d < (int)tree[i].dist.size(); d++)
+			{
+				cout << " " << tree[i].dist[d];
+			}
+			cout << endl;
 			if(!tree[i].isleaf && tree[i].left < 0)
 			{  
 				int splitfeat;
@@ -380,8 +438,16 @@ void SemSegContextTree::train ( const MultiDataset *md )
 				getBestSplit(allfeats, currentfeats,labels, i, splitfeat, splitval);
 				tree[i].feat = splitfeat;
 				tree[i].decision = splitval;
+				cout << "splitfeat: " << splitfeat << " splitval: " << splitval << endl;
 				if(splitfeat >= 0)
 				{
+					int x1 = featsel[splitfeat][0];
+					int y1 = featsel[splitfeat][1];
+					int x2 = featsel[splitfeat][2];
+					int y2 = featsel[splitfeat][3];
+					int p = featsel[splitfeat][4];
+					int o = featsel[splitfeat][5];
+
 					allleaf = false;
 					int left = tree.size();
 					tree.push_back(Node());
@@ -393,7 +459,8 @@ void SemSegContextTree::train ( const MultiDataset *md )
 					tree[right].dist = vector<double>(classes, 0.0);
 					tree[left].depth = depth+1;
 					tree[right].depth = depth+1;
-//#pragma omp parallel for
+					
+#pragma omp parallel for
 					for(int iCounter = 0; iCounter < imgcounter; iCounter++)
 					{
 						int xsize = currentfeats[iCounter].size();
@@ -404,7 +471,12 @@ void SemSegContextTree::train ( const MultiDataset *md )
 							{
 								if(currentfeats[iCounter][x][y] == i)
 								{
-									if(allfeats[iCounter][x][y][splitfeat] < splitval)
+									int xsize = (int)allfeats[iCounter].size();
+									int ysize = (int)allfeats[iCounter][0].size();
+									double val1 = allfeats[iCounter][BOUND(x+x1,0,xsize-1)][BOUND(y+y1,0,ysize-1)][p];
+									double val2 = allfeats[iCounter][BOUND(x+x2,0,xsize-1)][BOUND(y+y2,0,ysize-1)][p];
+									double val = ops[o]->getVal(val1,val2);
+									if(val < splitval)
 									{ 
 										currentfeats[iCounter][x][y] = left;
 										tree[left].dist[labelmap[labels[iCounter][x][y]]]++;
@@ -426,6 +498,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 						tree[right].dist[d]/=a[d];
 						rcounter +=tree[right].dist[d];
 					}
+					assert(lcounter > 0 && rcounter > 0);
 					for(uint d = 0; d < tree[left].dist.size(); d++)
 					{
 						tree[left].dist[d]/=lcounter;
@@ -475,7 +548,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 		
 	vector<vector<vector<double> > > feats;
 	
-#if 1
+#if 0
 	lfcw->getFeats(img, feats);
 #else
 	feats = vector<vector<vector<double> > >(xsize,vector<vector<double> >(ysize,vector<double>(3,0.0)));
@@ -511,7 +584,19 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 					if(tree[t].left > 0)
 					{
 						allleaf = false;
-						if(feats[x][y][tree[t].feat] < tree[t].decision)
+						int splitfeat = tree[t].feat;
+						int x1 = featsel[splitfeat][0];
+						int y1 = featsel[splitfeat][1];
+						int x2 = featsel[splitfeat][2];
+						int y2 = featsel[splitfeat][3];
+						int p = featsel[splitfeat][4];
+						int o = featsel[splitfeat][5];
+
+						double val1 = feats[BOUND(x+x1,0,xsize-1)][BOUND(y+y1,0,ysize-1)][p];
+						double val2 = feats[BOUND(x+x2,0,xsize-1)][BOUND(y+y2,0,ysize-1)][p];
+						double val = ops[o]->getVal(val1,val2);
+						
+						if(val < tree[t].decision)
 						{
 							currentfeats[x][y] = tree[t].left;
 						}

semseg/SemSegContextTree.h

@@ -52,6 +52,48 @@ class SemSegContextTree : public SemanticSegmentation
 		/** standard constructor */
 		Node(int _left, int _right, int _feat, double _decision):left(_left),right(_right),feat(_feat), decision(_decision),isleaf(false){}
 	};
+	
+	class Operation
+	{
+	public:
+		virtual double getVal(const double &v1, const double &v2) = 0;
+	};
+
+	class Minus:public Operation
+	{
+	public:
+		virtual double getVal(const double &v1, const double &v2)
+		{
+			return v1-v2;
+		}
+	};
+
+	class MinusAbs:public Operation
+	{
+	public:
+		virtual double getVal(const double &v1, const double &v2)
+		{
+			return abs(v1-v2);
+		}
+	};
+
+	class Addition:public Operation
+	{
+	public:
+		virtual double getVal(const double &v1, const double &v2)
+		{
+			return v1+v2;
+		}
+	};
+	
+	class Only1:public Operation
+	{
+	public:
+		virtual double getVal(const double &v1, const double &v2)
+		{
+			return v1;
+		}
+	};
       
 	/** store features */
 	VVector currentfeats;
@@ -71,6 +113,12 @@ class SemSegContextTree : public SemanticSegmentation
 	/** maximum samples for tree  */
 	int maxSamples;
 	
+	/** size for neighbourhood */
+	int windowSize;
+	
+	/** how many feats should be considered for a split */
+	int featsPerSplit;
+	
 	/** count samples per label */
 	map<int,int> labelcounter;
 	
@@ -89,6 +137,15 @@ class SemSegContextTree : public SemanticSegmentation
 	/** maximal depth of tree */
 	int maxDepth;
 	
+	/** operations for pairwise features */
+	vector<Operation*> ops;
+	
+	/** number of possible features */
+	int allfeatsize;
+	
+	/** vector of all possible features */
+	vector<vector<int> > featsel;
+	
     public:
 	/** simple constructor */
 	SemSegContextTree( const Config *conf, const MultiDataset *md );