segmentation fixed and some new features

semseg/SemSegContextTree.cpp

@@ -22,6 +22,8 @@ using namespace OBJREC;
 using namespace std;
 using namespace NICE;
 
+
+
 class MCImageAccess:public ValueAccess
 {
 public:
@@ -69,7 +71,15 @@ public:
 	
 	virtual string writeInfos()
 	{
-		return "Minus"+values->writeInfos();
+		string out = "Minus";
+		if(values !=NULL)
+			out+=values->writeInfos();
+		return out;
+	}
+	
+	virtual OperationTypes getOps()
+	{
+		return MINUS;
 	}
 };
 
@@ -92,7 +102,15 @@ public:
 	
 	virtual string writeInfos()
 	{
-		return "MinusAbs"+values->writeInfos();
+		string out = "MinusAbs";
+		if(values !=NULL)
+			out+=values->writeInfos();
+		return out;
+	}
+	
+	virtual OperationTypes getOps()
+	{
+		return MINUSABS;
 	}
 };
 
@@ -115,7 +133,15 @@ public:
 	
 	virtual string writeInfos()
 	{
-		return "Addition"+values->writeInfos();
+		string out = "Addition";
+		if(values !=NULL)
+			out+=values->writeInfos();
+		return out;
+	}
+	
+	virtual OperationTypes getOps()
+	{
+		return ADDITION;
 	}
 };
 
@@ -137,7 +163,15 @@ public:
 	
 	virtual string writeInfos()
 	{
-		return "Only1"+values->writeInfos();
+		string out = "Only1";
+		if(values !=NULL)
+			out+=values->writeInfos();
+		return out;
+	}
+	
+	virtual OperationTypes getOps()
+	{
+		return ONLY1;
 	}
 };
 
@@ -170,6 +204,8 @@ public:
 		double val3 = intImg.get(uLx,lRy, chan);
 		double val4 = intImg.get(lRx,lRy, chan);
 		double area = (lRx-uLx)*(lRy-uLy);
+		if(area == 0)
+			return 0.0;
 		return (val1+val4-val2-val3)/area;
 	}
 	
@@ -182,13 +218,18 @@ public:
 	{
 		return "IntegralOps";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return INTEGRAL;
+	}
 };
 
 //uses mean of Integral image given by x1, y1 with current pixel as center
 class IntegralCenteredOps:public IntegralOps
 {
 public:
-	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2)
+	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values)
 	{
 		x1 = abs(_x1);
 		y1 = abs(_y1);
@@ -196,6 +237,7 @@ public:
 		y2 = abs(_y2);
 		channel1 = _channel1;
 		channel2 = _channel2;
+		values = _values;
 	}
 	
 	virtual double getVal(const Features &feats, const int &x, const int &y)
@@ -214,13 +256,18 @@ public:
 	{
 		return "IntegralCenteredOps";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return INTEGRALCENT;
+	}
 };
 
 //uses different of mean of Integral image given by two windows, where (x1,y1) is the width and height of window1 and (x2,y2) of window 2
 class BiIntegralCenteredOps:public IntegralCenteredOps
 {
 public:
-	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2)
+		virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values)
 	{
 		x1 = min(abs(_x1),abs(_x2));
 		y1 = min(abs(_y1),abs(_y2));
@@ -228,6 +275,7 @@ public:
 		y2 = max(abs(_y1),abs(_y2));
 		channel1 = _channel1;
 		channel2 = _channel2;
+		values = _values;
 	}
 	
 	virtual double getVal(const Features &feats, const int &x, const int &y)
@@ -246,6 +294,11 @@ public:
 	{
 		return "BiIntegralCenteredOps";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return BIINTEGRALCENT;
+	}
 };
 
 /** horizontal Haar features
@@ -267,10 +320,20 @@ class HaarHorizontal:public IntegralCenteredOps
 		return computeMean(*feats.integralImg,tlx,tly,lrx, y,channel1)-computeMean(*feats.integralImg,tlx,y,lrx, lry,channel1);
 	}
 	
+	virtual Operation* clone()
+	{
+		return new HaarHorizontal();
+	}
+	
 	virtual string writeInfos()
 	{
 		return "HaarHorizontal";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return HAARHORIZ;
+	}
 };
 
 /** vertical Haar features
@@ -292,10 +355,20 @@ class HaarVertical:public IntegralCenteredOps
 		return computeMean(*feats.integralImg,tlx,tly,x, lry,channel1)-computeMean(*feats.integralImg,x,tly,lrx, lry,channel1);
 	}
 	
+	virtual Operation* clone()
+	{
+		return new HaarVertical();
+	}
+	
 	virtual string writeInfos()
 	{
 		return "HaarVertical";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return HAARVERT;
+	}
 };
 
 /** vertical Haar features
@@ -317,10 +390,20 @@ class HaarDiag:public IntegralCenteredOps
 		return computeMean(*feats.integralImg,tlx,tly,x, y,channel1)+computeMean(*feats.integralImg,x,y,lrx, lry,channel1) - computeMean(*feats.integralImg,tlx,y,x, lry,channel1)-computeMean(*feats.integralImg,x,tly,lrx, y,channel1);
 	}
 	
+	virtual Operation* clone()
+	{
+		return new HaarDiag();
+	}
+	
 	virtual string writeInfos()
 	{
 		return "HaarDiag";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return HAARDIAG;
+	}
 };
 
 /** horizontal Haar features
@@ -338,17 +421,27 @@ class Haar3Horiz:public BiIntegralCenteredOps
 		int tlx = BOUND(x-x2,0,xsize-1);
 		int tly = BOUND(y-y2,0,ysize-1);
 		int mtly = BOUND(y-y1,0,ysize-1);
-		int mlry = BOUND(y-y1,0,ysize-1);
+		int mlry = BOUND(y+y1,0,ysize-1);
 		int lrx = BOUND(x+x2,0,xsize-1);
 		int lry = BOUND(y+y2,0,ysize-1);
 		
 		return computeMean(*feats.integralImg,tlx,tly,lrx, mtly,channel1) -computeMean(*feats.integralImg,tlx,mtly,lrx, mlry,channel1) + computeMean(*feats.integralImg,tlx,mlry,lrx, lry,channel1);
 	}
 	
+	virtual Operation* clone()
+	{
+		return new Haar3Horiz();
+	}
+	
 	virtual string writeInfos()
 	{
 		return "Haar3Horiz";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return HAAR3HORIZ;
+	}
 };
 
 /** vertical Haar features
@@ -366,17 +459,27 @@ class Haar3Vert:public BiIntegralCenteredOps
 		int tlx = BOUND(x-x2,0,xsize-1);
 		int tly = BOUND(y-y2,0,ysize-1);
 		int mtlx = BOUND(x-x1,0,xsize-1);
-		int mlrx = BOUND(x-x1,0,xsize-1);
+		int mlrx = BOUND(x+x1,0,xsize-1);
 		int lrx = BOUND(x+x2,0,xsize-1);
 		int lry = BOUND(y+y2,0,ysize-1);
 		
 		return computeMean(*feats.integralImg,tlx,tly,mtlx, lry,channel1) -computeMean(*feats.integralImg,mtlx,tly,mlrx, lry,channel1) + computeMean(*feats.integralImg,mlrx,tly,lrx, lry,channel1);
 	}
 	
+	virtual Operation* clone()
+	{
+		return new Haar3Vert();
+	}
+	
 	virtual string writeInfos()
 	{
 		return "Haar3Vert";
 	}
+	
+	virtual OperationTypes getOps()
+	{
+		return HAAR3VERT;
+	}
 };
 
 SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md )
@@ -440,6 +543,8 @@ SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md
 	cops.push_back(new Haar3Horiz());
 	cops.push_back(new Haar3Vert());
 	
+	opOverview = vector<int>(NBOPERATIONS, 0);
+	
 	calcVal.push_back(new MCImageAccess());
 	calcVal.push_back(new ClassificationResultAcess());
 	
@@ -571,7 +676,7 @@ double SemSegContextTree::getBestSplit(std::vector<NICE::MultiChannelImageT<doub
 		
 		if(ft > 0)
 		{
-			tmpws *= 2;
+			tmpws *= 4;
 		}
 		
 		if(useGaussian)
@@ -601,19 +706,25 @@ double SemSegContextTree::getBestSplit(std::vector<NICE::MultiChannelImageT<doub
 		}
 		else if(ft == 1)
 		{
-			int chans = integralImgs[0].channels;
+
 			int opssize = (int)ops.size();
-			int f1 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
-			int f2 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
-			int o = (int)((double)rand()/(double)RAND_MAX*((double)cops.size())+(double)opssize);
+			//opssize = 0;
+			int o = (int)((double)rand()/(double)RAND_MAX*(((double)cops.size())+(double)opssize));
+			
 			Operation *op;
 			if(o < opssize)
 			{
+				int chans = (int)forest[0][0].dist.size();
+				int f1 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
+				int f2 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
 				op = ops[o]->clone();
 				op->set(x1,y1,x2,y2,f1,f2, calcVal[ft]);
 			}
 			else
 			{
+				int chans = integralImgs[0].channels();
+				int f1 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
+				int f2 = (int)((double)rand()/(double)RAND_MAX*(double)chans);
 				o -= opssize;
 				op = cops[o]->clone();
 				op->set(x1,y1,x2,y2,f1,f2, calcVal[ft]);
@@ -621,7 +732,6 @@ double SemSegContextTree::getBestSplit(std::vector<NICE::MultiChannelImageT<doub
 			featsel.push_back(op);
 		}
 	}
-	
 #pragma omp parallel for private(mapit)
 	for(int f = 0; f < featsPerSplit; f++)
 	{
@@ -738,12 +848,13 @@ inline double SemSegContextTree::getMeanProb(const int &x,const int &y,const int
 	return val / (double)nbTrees;
 }
 
-void SemSegContextTree::computeIntegralImage(const NICE::MultiChannelImageT<int> &currentfeats, const NICE::MultiChannelImageT<int> &lfeats, NICE::MultiChannelImageT<double> &integralImage)
+void SemSegContextTree::computeIntegralImage(const NICE::MultiChannelImageT<int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage)
 {
 	int xsize = currentfeats.width();
 	int ysize = currentfeats.height();
 	
 	int channels = (int)forest[0][0].dist.size();
+
 #pragma omp parallel for
 	for(int c = 0; c < channels; c++)
 	{
@@ -772,12 +883,16 @@ void SemSegContextTree::computeIntegralImage(const NICE::MultiChannelImageT<int>
 		}
 	}
 	
-	int channels2 = (int)lfeats.size();
-	if(lfeats.get(xsize-1,ysize-1,0) == 0)
+	int channels2 = (int)lfeats.channels();
+	
+	xsize = lfeats.width();
+	ysize = lfeats.height();
+	if(integralImage.get(xsize-1,ysize-1,channels) == 0.0)
 	{
 #pragma omp parallel for
-		for(int c = channels, int c1 = 0; c1 < channels2; c++, c1++)
+		for(int c1 = 0; c1 < channels2; c1++)
 		{
+			int c = channels+c1; 
 			integralImage.set(0,0,lfeats.get(0,0,c1), c);
 			
 			//first column
@@ -828,6 +943,33 @@ void SemSegContextTree::train ( const MultiDataset *md )
 	
 	int imgcounter = 0;
 	
+	/*MultiChannelImageT<int> ttmp2(0,0,0);
+	MultiChannelImageT<double> ttmp1(100,100,1);
+	MultiChannelImageT<double> tint(100,100,1);
+	ttmp1.setAll(1.0);
+	tint.setAll(0.0);
+	computeIntegralImage(ttmp2,ttmp1,tint);
+
+	
+	for(int i = 0; i < cops.size(); i++)
+	{
+		Features feats;
+		feats.feats = &tint;
+		feats.cfeats = &ttmp2;
+		feats.cTree = 0;
+		feats.tree = new vector<TreeNode>;
+		feats.integralImg = &tint;
+		cops[i]->set(-10, -6, 8, 9, 0, 0, new MCImageAccess());
+		cout << "for: " << cops[i]->writeInfos() << endl;
+		int y = 50;
+		for(int x = 40; x < 44; x++)
+		{
+			cout << "x: " << x << " val: " << cops[i]->getVal(feats, x, y) << endl;
+		}
+	}
+	
+	getchar();*/
+	
 	LOOP_ALL_S ( *trainp )
 	{
 		EACH_INFO ( classno,info );
@@ -1058,8 +1200,39 @@ void SemSegContextTree::train ( const MultiDataset *md )
 							}
 						}
 						
-						assert(lcounter > 0 && rcounter > 0);
-
+						if(lcounter <= 0 || rcounter <= 0)
+						{
+							cout << "lcounter : " << lcounter << " rcounter: " << rcounter << endl;
+							cout << "splitval: " << splitval << " splittype: " << splitfeat->writeInfos() << endl;
+							cout << "bestig: " << bestig << endl;
+							for(int iCounter = 0; iCounter < imgcounter; iCounter++)
+							{
+								int xsize = currentfeats[iCounter].width();
+								int ysize = currentfeats[iCounter].height();
+								int counter = 0;
+								for(int x = 0; x < xsize; x++)
+								{
+									for(int y = 0; y < ysize; y++)
+									{
+										if(lastfeats[iCounter].get(x,y,tree) == i)
+										{
+											if(++counter > 30)
+												break;
+											Features feat;
+											feat.feats = &allfeats[iCounter];
+											feat.cfeats = &lastfeats[iCounter];
+											feat.cTree = tree;
+											feat.tree = &forest[tree];
+											feat.integralImg = &integralImgs[iCounter];
+											double val = splitfeat->getVal(feat,x,y);
+											cout << "splitval: " << splitval << " val: " << val << endl;
+										}
+									}
+								}
+							}
+							assert(lcounter > 0 && rcounter > 0);
+						}
+						
 						for(uint d = 0; d < forest[tree][left].dist.size(); d++)
 						{
 							forest[tree][left].dist[d]/=lcounter;
@@ -1076,7 +1249,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		//TODO: features neu berechnen!
 			
 		//compute integral image
-		int channels = classes+allfeats.size();
+		int channels = classes+allfeats[0].channels();
 			
 		if(integralImgs[0].width() == 0)
 		{
@@ -1085,6 +1258,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 				int xsize = allfeats[i].width();
 				int ysize = allfeats[i].height();
 				integralImgs[i].reInit(xsize, ysize, channels);
+				integralImgs[i].setAll(0.0);
 			}
 		}
 			
@@ -1102,7 +1276,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
 	#endif
 	}
 	
-	
 #ifdef DEBUG
 	for(int tree = 0; tree < nbTrees; tree++)
 	{
@@ -1111,7 +1284,10 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		{
 			printf("tree[%i]: left: %i, right: %i", i, forest[tree][i].left, forest[tree][i].right);
 			if(!forest[tree][i].isleaf && forest[tree][i].left != -1)
+			{
 				cout <<  ", feat: " << forest[tree][i].feat->writeInfos() << " ";
+				opOverview[forest[tree][i].feat->getOps()]++;
+			}
 			for(int d = 0; d < (int)forest[tree][i].dist.size(); d++)
 			{
 				cout << " " << forest[tree][i].dist[d];
@@ -1119,6 +1295,16 @@ void SemSegContextTree::train ( const MultiDataset *md )
 			cout << endl;
 		}
 	}
+	
+	for(uint c = 0; c < ops.size(); c++)
+	{
+		cout << ops[c]->writeInfos() << ": " << opOverview[ops[c]->getOps()] << endl;
+	}
+	
+	for(uint c = 0; c < cops.size(); c++)
+	{
+		cout << cops[c]->writeInfos() << ": " << opOverview[cops[c]->getOps()] << endl;
+	}
 #endif
 }
 
@@ -1210,7 +1396,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 			}
 			
 			//compute integral image
-			int channels = (int)labelmap.size()+feats.size();
+			int channels = (int)labelmap.size()+feats.channels();
 			
 			if(integralImg.width() == 0)
 			{
@@ -1246,6 +1432,8 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 					}
 					segresult.setPixel(x,y,maxindex);
 				}
+				if(maxvalue > 1)
+					cout << "maxvalue: " << maxvalue << endl;
 			}
 		}
 	}
@@ -1256,10 +1444,29 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 		Matrix regions;
 		int regionNumber = segmentation->segRegions(img,regions);
 		cout << "regions: " << regionNumber << endl;
-		int dSize = (int)labelmap.size();
+		int dSize = forest[0][0].dist.size();
 		vector<vector<double> > regionProbs(regionNumber, vector<double>(dSize,0.0));
 		vector<int> bestlabels(regionNumber, 0);
 		
+		/*
+		for(int r = 0; r < regionNumber; r++)
+		{
+			Image over(img.width(), img.height());
+			for(int y = 0; y < img.height(); y++)
+			{
+				for(int x = 0; x < img.width(); x++)
+				{
+					if(((int)regions(x,y)) == r)
+						over.setPixel(x,y,1);
+					else
+						over.setPixel(x,y,0);
+				}
+			}
+			cout << "r: " << r << endl;
+			showImageOverlay(img, over);
+		}
+		*/
+		
 		for(int y = 0; y < img.height(); y++)
 		{
 			for(int x = 0; x < img.width(); x++)
@@ -1272,9 +1479,17 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 			}
 		}
 		
+		int roi = 38;
+		
 		for(int r = 0; r < regionNumber; r++)
 		{
 			double maxval = regionProbs[r][0];
+			bestlabels[r] = 0;
+			if(roi == r)
+			{
+				cout << "r: " << r << endl;
+				cout << "0: " << regionProbs[r][0] << endl;
+			}
 			for(int d = 1; d < dSize; d++)
 			{
 				if(maxval < regionProbs[r][d])
@@ -1282,6 +1497,14 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 					maxval = regionProbs[r][d];
 					bestlabels[r] = d;
 				}
+				if(roi == r)
+				{
+					cout << d << ": " << regionProbs[r][d] << endl;
+				}
+			}
+			if(roi == r)
+			{
+				cout << "bestlabel: " << bestlabels[r] << " danach: " << labelmapback[bestlabels[r]] << endl;
 			}
 			bestlabels[r] = labelmapback[bestlabels[r]];
 		}
@@ -1290,6 +1513,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 		{
 			for(int x = 0; x < img.width(); x++)
 			{
+				
 				segresult.setPixel(x,y,bestlabels[regions(x,y)]);
 			}
 		}

semseg/SemSegContextTree.h

@@ -66,12 +66,34 @@ public:
 	virtual std::string writeInfos() = 0;
 };
 
+enum OperationTypes { 
+	MINUS, 
+	MINUSABS, 
+	ADDITION,
+	ONLY1,
+	INTEGRAL,
+	INTEGRALCENT,
+	BIINTEGRALCENT,
+	HAARHORIZ,
+	HAARVERT,
+	HAARDIAG,
+	HAAR3HORIZ,
+	HAAR3VERT,
+	NBOPERATIONS
+};
+
 class Operation
 {
 protected:
 	int x1, y1, x2, y2, channel1, channel2;
 	ValueAccess *values;
 public:
+	
+	Operation()
+	{
+		values = NULL;
+	}
+	
 	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2, ValueAccess *_values)
 	{
 		x1 = _x1;
@@ -100,6 +122,8 @@ public:
 		xsize = feats.feats->width();
 		ysize = feats.feats->height();
 	}
+	
+	virtual OperationTypes getOps() = 0;
 };
   
 /** Localization system */
@@ -141,6 +165,9 @@ class SemSegContextTree : public SemanticSegmentation
 	/** scalefactor for balancing for each class */
 	std::vector<double> a;
 	
+	/** counter for used operations */
+	std::vector<int> opOverview;
+	
 	/** the minimum number of features allowed in a leaf */
 	int minFeats;
 	
@@ -208,7 +235,7 @@ class SemSegContextTree : public SemanticSegmentation
 	 * @param integralImage output image (must be initilized)
 	 * @return void
 	 **/
-	void computeIntegralImage(const NICE::MultiChannelImageT<int> &currentfeats, const NICE::MultiChannelImageT<int> &lfeats, NICE::MultiChannelImageT<double> &integralImage);
+	void computeIntegralImage(const NICE::MultiChannelImageT<int> &currentfeats, const NICE::MultiChannelImageT<double> &lfeats, NICE::MultiChannelImageT<double> &integralImage);
 	
 	/**
 	 * compute best split for current settings