neue Kontextmerkmale

semseg/SemSegContextTree.cpp

@@ -24,7 +24,7 @@ using namespace NICE;
 class Minus:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -47,7 +47,7 @@ public:
 class MinusAbs:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -70,7 +70,7 @@ public:
 class Addition:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -93,7 +93,7 @@ public:
 class Only1:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -115,7 +115,7 @@ public:
 class ContextMinus:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -138,7 +138,7 @@ public:
 class ContextMinusAbs:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -161,7 +161,7 @@ public:
 class ContextAddition:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -184,7 +184,7 @@ public:
 class ContextOnly1:public Operation
 {
 public:
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
 		int xsize = feats.width();
 		int ysize = feats.height();
@@ -217,10 +217,21 @@ public:
 		channel2 = _channel2;
 	}
 	
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y)
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
 	{
-		MultiChannelImageT<double> intImg;
-		return computeMean(intImg,x1,y1,x2,y2,channel1);
+		int xsize = feats.width();
+		int ysize = feats.height();
+		return computeMean(integralImg,BOUND(x+x1,0,xsize-1),BOUND(y+y1,0,ysize-1),BOUND(x+x2,0,xsize-1),BOUND(y+y2,0,ysize-1),channel1);
+	}
+	
+	inline double computeMean(const NICE::MultiChannelImageT<double> &intImg, const int &uLx, const int &uLy, const int &lRx, const int &lRy, const int &chan)
+	{
+		double val1 = intImg.get(uLx,uLy, chan);
+		double val2 = intImg.get(lRx,uLy, chan);
+		double val3 = intImg.get(uLx,lRy, chan);
+		double val4 = intImg.get(lRx,lRy, chan);
+		double area = (lRx-uLx)*(lRy-uLy);
+		return (val1+val4-val2-val3)/area;
 	}
 	
 	virtual Operation* clone()
@@ -232,16 +243,6 @@ public:
 	{
 		return "IntegralOps";
 	}
-	
-	inline double computeMean(const NICE::MultiChannelImageT<double> &intImg, int &uLx, int &uLy, int &lRx, int &lRy, int &chan)
-	{
-		double val1 = intImg.get(uLx,uLy, chan);
-		double val2 = intImg.get(lRx,uLy, chan);
-		double val3 = intImg.get(uLx,lRy, chan);
-		double val4 = intImg.get(lRx,lRy, chan);
-		double area = (lRx-uLx)*(lRy-uLy);
-		return (val1+val4-val2-val3)/area;
-	}
 };
 
 //uses mean of Integral image given by x1, y1 with current pixel as center
@@ -250,10 +251,12 @@ class IntegralCenteredOps:public IntegralOps
 public:
 	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2)
 	{
-		x1 = _x1;
-		y1 = _y1;
+		x1 = min(_x1,-_x1);
+		y1 = min(_y1,-_y1);
 		x2 = -x1;
 		y2 = -y1;
+		channel1 = _channel1;
+		channel2 = _channel2;
 	}
 	
 	virtual Operation* clone()
@@ -265,12 +268,39 @@ public:
 	{
 		return "IntegralCenteredOps";
 	}
-
 };
 
 //uses different of mean of Integral image given by two windows, where (x1,y1) is the width and height of window1 and (x2,y2) of window 2
-//TODO von Integral ops ableiten
-
+class BiIntegralCenteredOps:public IntegralOps
+{
+public:
+	virtual void set(int _x1, int _y1, int _x2, int _y2, int _channel1, int _channel2)
+	{
+		x1 = min(abs(_x1),abs(_x2));
+		y1 = min(abs(_y1),abs(_y2));
+		x2 = max(abs(_x1),abs(_x2));
+		y2 = max(abs(_y1),abs(_y2));
+		channel1 = _channel1;
+		channel2 = _channel2;
+	}
+	
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, MultiChannelImageT<double>  &integralImg, const int &x, const int &y)
+	{
+		int xsize = feats.width();
+		int ysize = feats.height();
+		return computeMean(integralImg,BOUND(x-x1,0,xsize-1),BOUND(y-y1,0,ysize-1),BOUND(x+x1,0,xsize-1),BOUND(y+y1,0,ysize-1),channel1) - computeMean(integralImg,BOUND(x-x2,0,xsize-1),BOUND(y-y2,0,ysize-1),BOUND(x+x2,0,xsize-1),BOUND(y+y2,0,ysize-1),channel1);
+	}
+	
+	virtual Operation* clone()
+	{
+		return new BiIntegralCenteredOps();
+	}
+	
+	virtual string writeInfos()
+	{
+		return "BiIntegralCenteredOps";
+	}
+};
 
 SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md )
     : SemanticSegmentation ( conf, &(md->getClassNames("train")) )
@@ -323,6 +353,9 @@ SemSegContextTree::SemSegContextTree( const Config *conf, const MultiDataset *md
 	cops.push_back(new ContextMinusAbs());
 	cops.push_back(new ContextAddition());
 	cops.push_back(new ContextOnly1());
+	cops.push_back(new BiIntegralCenteredOps());
+	cops.push_back(new IntegralCenteredOps());
+	cops.push_back(new IntegralOps());
 	
 	classnames = md->getClassNames ( "train" );
 	
@@ -337,7 +370,7 @@ SemSegContextTree::~SemSegContextTree()
 {
 }
 
-void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &feats, vector<vector<vector<int> > > &currentfeats,const vector<vector<vector<int> > > &labels, int node, Operation *&splitop, double &splitval)
+void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &feats, vector<vector<vector<int> > > &currentfeats, vector<MultiChannelImageT<double> > &integralImgs, const vector<vector<vector<int> > > &labels, int node, Operation *&splitop, double &splitval)
 {
 
 	int imgCount = 0, featdim = 0;
@@ -439,7 +472,6 @@ void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &
 		cout << "globent to small: " << globent << endl;
 		return;
 	}
-	
 
 	int classes = (int)labelmap.size();
 	featsel.clear();
@@ -464,6 +496,9 @@ void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &
 		}
 		
 		int ft = (int)((double)rand()/(double)RAND_MAX*(double)ftypes);
+
+		if(integralImgs[0].width() == 0)
+			ft = 0;
 		
 		if(ft == 0)
 		{
@@ -483,7 +518,6 @@ void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &
 			op->set(x1,y1,x2,y2,f1,f2);
 			featsel.push_back(op);
 		}
-		
 	}
 	
 #pragma omp parallel for private(mapit)
@@ -496,7 +530,7 @@ void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &
 		
 		for ( it=selFeats.begin() ; it != selFeats.end(); it++ )
 		{
-			vals.push_back(featsel[f]->getVal(feats[(*it)[0]],currentfeats[(*it)[0]],tree,(*it)[1], (*it)[2]));
+			vals.push_back(featsel[f]->getVal(feats[(*it)[0]],currentfeats[(*it)[0]],tree, integralImgs[(*it)[0]], (*it)[1], (*it)[2]));
 		}
 		
 		int counter = 0;
@@ -584,6 +618,42 @@ void SemSegContextTree::getBestSplit(const vector<MultiChannelImageT<double> > &
 #endif
 }
 
+void SemSegContextTree::computeIntegralImage(const vector<vector<int> > &currentfeats, MultiChannelImageT<double> &integralImage)
+{
+	int xsize = currentfeats.size();
+	assert(xsize > 0);
+	int ysize = currentfeats[0].size();
+	
+	int channels = (int)labelmap.size();
+			
+	for(int c = 0; c < channels; c++)
+	{
+		integralImage.set(0,0,tree[currentfeats[0][0]].dist[c], c);
+		
+		//first column
+		for(int y = 1; y < ysize; y++)
+		{
+			integralImage.set(0,0,tree[currentfeats[0][y]].dist[c]+integralImage.get(0,y,c), c);
+		}
+		
+		//first row
+		for(int x = 1; x < xsize; x++)
+		{
+			integralImage.set(0,0,tree[currentfeats[x][0]].dist[c]+integralImage.get(x,0,c), c);
+		}
+		
+		//rest
+		for(int y = 1; y < ysize; y++)
+		{
+			for(int x = 1; x < xsize; x++)
+			{
+				double val = tree[currentfeats[x][y]].dist[c]+integralImage.get(x,y-1,c)+integralImage.get(x-1,y,c)-integralImage.get(x-1,y-1,c);
+				integralImage.set(0, 0, val, c);
+			}
+		}
+	}
+}
+
 void SemSegContextTree::train ( const MultiDataset *md )
 {
 	const LabeledSet train = * ( *md ) ["train"];
@@ -733,6 +803,8 @@ void SemSegContextTree::train ( const MultiDataset *md )
 	bool allleaf = false;
 	//int baseFeatSize = allfeats[0].size();
 	
+	vector<MultiChannelImageT<double> > integralImgs(imgcounter,MultiChannelImageT<double>());
+	
 	while(!allleaf && depth < maxDepth)
 	{
 		allleaf = true;
@@ -743,8 +815,6 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		startnode = t;
 		vector<vector<vector<int> > > lastfeats = currentfeats;
 		
-		vector<MultiChannelImageT<double> > integralImgs;
-		
 //#pragma omp parallel for
 		for(int i = s; i < t; i++)
 		{
@@ -752,7 +822,8 @@ void SemSegContextTree::train ( const MultiDataset *md )
 			{  
 				Operation *splitfeat = NULL;
 				double splitval;
-				getBestSplit(allfeats, lastfeats,labels, i, splitfeat, splitval);
+				
+				getBestSplit(allfeats, lastfeats, integralImgs, labels, i, splitfeat, splitval);
 				tree[i].feat = splitfeat;
 				
 				tree[i].decision = splitval;
@@ -781,7 +852,7 @@ void SemSegContextTree::train ( const MultiDataset *md )
 							{
 								if(currentfeats[iCounter][x][y] == i)
 								{
-									double val = splitfeat->getVal(allfeats[iCounter],lastfeats[iCounter],tree,x,y);
+									double val = splitfeat->getVal(allfeats[iCounter],lastfeats[iCounter], tree, integralImgs[iCounter],x,y);
 									if(val < splitval)
 									{ 
 										currentfeats[iCounter][x][y] = left;
@@ -834,55 +905,34 @@ void SemSegContextTree::train ( const MultiDataset *md )
 		}
 		//TODO: features neu berechnen!
 		
-#if 1
 		//compute integral image
 		int channels = (int)labelmap.size();
 		
-		if(integralImgs.size() == 0)
+		if(integralImgs[0].width() == 0)
 		{
 			for(int i = 0; i < imgcounter; i++)
 			{
 				int xsize = allfeats[i].width();
 				int ysize = allfeats[i].height();
 				
-				integralImgs.push_back(MultiChannelImageT<double>(xsize, ysize, channels));
+				integralImgs[i].reInit(xsize, ysize, channels);
 			}
 		}
 		
 		for(int i = 0; i < imgcounter; i++)
 		{
-			int xsize = allfeats[i].width();
-			int ysize = allfeats[i].height();
-			
-			for(int c = 0; c < channels; c++)
+			computeIntegralImage(currentfeats[i],integralImgs[i]);
+		}
+		
+		if(depth == 4)
+		{
+			cout << "learn: ";
+			for(int x = 20; x < 30; x++)
 			{
-				integralImgs[i].set(0,0,tree[currentfeats[i][0][0]].dist[c], c);
-				
-				//first column
-				for(int y = 1; y < ysize; y++)
-				{
-					integralImgs[i].set(0,0,tree[currentfeats[i][0][y]].dist[c]+integralImgs[i].get(0,y,c), c);
-				}
-				
-				//first row
-				for(int x = 1; x < xsize; x++)
-				{
-					integralImgs[i].set(0,0,tree[currentfeats[i][x][0]].dist[c]+integralImgs[i].get(x,0,c), c);
-				}
-				
-				//rest
-				for(int y = 1; y < ysize; y++)
-				{
-					for(int x = 1; x < xsize; x++)
-					{
-						double val = tree[currentfeats[i][x][y]].dist[c]+integralImgs[i].get(x,y-1,c)+integralImgs[i].get(x-1,y,c)-integralImgs[i].get(x-1,y-1,c);
-						integralImgs[i].set(0, 0, val, c);
-					}
-				}
+				cout << currentfeats[0][x][20] << " ";
 			}
+			cout << endl;
 		}
-#endif
-		
 		
 		depth++;
 #ifdef DEBUG
@@ -952,6 +1002,8 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 	
 	bool allleaf = false;
 	
+	MultiChannelImageT<double> integralImg;
+	
 	vector<vector<int> > currentfeats = vector<vector<int> >(xsize, vector<int>(ysize,0));
 	int depth = 0;
 	while(!allleaf)
@@ -968,7 +1020,7 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 				if(tree[t].left > 0)
 				{
 					allleaf = false;
-					double val = tree[t].feat->getVal(feats,lastfeats,tree,x,y);
+					double val = tree[t].feat->getVal(feats,lastfeats,tree,integralImg,x,y);
 					
 					if(val < tree[t].decision)
 					{
@@ -982,7 +1034,28 @@ void SemSegContextTree::semanticseg ( CachedExample *ce, NICE::Image & segresult
 			}
 		}
 		
-		//TODO: features neu berechnen! analog zum training
+		//compute integral image
+		int channels = (int)labelmap.size();
+		
+		if(integralImg.width() == 0)
+		{
+			int xsize = feats.width();
+			int ysize = feats.height();
+			
+			integralImg.reInit(xsize, ysize, channels);
+		}
+		
+		computeIntegralImage(currentfeats,integralImg);
+		
+		if(depth == 4)
+		{
+			cout << "learn: ";
+			for(int x = 20; x < 30; x++)
+			{
+				cout << currentfeats[x][20] << " ";
+			}
+			cout << endl;
+		}
 		
 		depth++;
 	}

semseg/SemSegContextTree.h

@@ -74,7 +74,7 @@ public:
 	 * @param y current y position
 	 * @return double distance
 	 **/
-	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, const int &x, const int &y) = 0;
+	virtual double getVal(const NICE::MultiChannelImageT<double> &feats, const std::vector<std::vector<int> > &cfeats, const std::vector<TreeNode> &tree, NICE::MultiChannelImageT<double> &integralImg, const int &x, const int &y) = 0;
 	virtual Operation* clone() = 0;
 	virtual std::string writeInfos() = 0;
 };
@@ -173,6 +173,16 @@ class SemSegContextTree : public SemanticSegmentation
 	 */
 	void train ( const MultiDataset *md );
 	
+	
+	/**
+	 * @brief computes integral image of given feats
+	 *
+	 * @param currentfeats input features
+	 * @param integralImage output image (must be initilized)
+	 * @return void
+	 **/
+	void computeIntegralImage(const std::vector<std::vector<int> > &currentfeats, MultiChannelImageT<double> &integralImage);
+	
 	/**
 	 * compute best split for current settings
 	 * @param feats features
@@ -182,7 +192,7 @@ class SemSegContextTree : public SemanticSegmentation
 	 * @param splitfeat output feature position
 	 * @param splitval 
 	 */
-	void getBestSplit(const std::vector<MultiChannelImageT<double> > &feats, std::vector<std::vector<std::vector<int> > > &currentfeats,const std::vector<std::vector<std::vector<int> > > &labels, int node, Operation *&splitfeat, double &splitval);
+	void getBestSplit(const std::vector<NICE::MultiChannelImageT<double> > &feats, std::vector<std::vector<std::vector<int> > > &currentfeats, std::vector<NICE::MultiChannelImageT<double> > &integralImgs, const std::vector<std::vector<std::vector<int> > > &labels, int node, Operation *&splitfeat, double &splitval);
 
 };