/******************************************************************** ** Image Component Library (ICL) ** ** ** ** Copyright (C) 2006-2013 CITEC, University of Bielefeld ** ** Neuroinformatics Group ** ** Website: www.iclcv.org and ** ** http://opensource.cit-ec.de/projects/icl ** ** ** ** File : ICLGeom/src/ICLGeom/SegmenterUtils.cpp ** ** Module : ICLGeom ** ** Authors: Andre Ueckermann ** ** ** ** ** ** GNU LESSER GENERAL PUBLIC LICENSE ** ** This file may be used under the terms of the GNU Lesser General ** ** Public License version 3.0 as published by the ** ** ** ** Free Software Foundation and appearing in the file LICENSE.LGPL ** ** included in the packaging of this file. Please review the ** ** following information to ensure the license requirements will ** ** be met: http://www.gnu.org/licenses/lgpl-3.0.txt ** ** ** ** The development of this software was supported by the ** ** Excellence Cluster EXC 277 Cognitive Interaction Technology. ** ** The Excellence Cluster EXC 277 is a grant of the Deutsche ** ** Forschungsgemeinschaft (DFG) in the context of the German ** ** Excellence Initiative. ** ** ** ********************************************************************/ #define __CL_ENABLE_EXCEPTIONS //enables openCL error catching #include #include #include #include #ifdef ICL_HAVE_OPENCL #include #include #include #endif namespace icl{ namespace geom{ #ifdef ICL_HAVE_OPENCL //OpenCL kernel code static char utilsKernel[] = " #pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable \n" "__kernel void \n" "segmentColoring(__global int const * assignment, __global uchar * colorR, __global uchar * colorG, __global uchar * colorB) \n" "{ \n" " size_t id = get_global_id(0); \n" " if(assignment[id]==0) \n" " { \n" " colorR[id]=128; \n" " colorG[id]=128; \n" " colorB[id]=128; \n" " } \n" " else \n" " { \n" " int H=(int)(assignment[id]*35.)%360; \n" " float S=1.0-assignment[id]*0.01; \n" " float hi=floor((float)H/60.); \n" " float f=((float)H/60.)-hi; \n" " float pp=1.0-S; \n" " float qq=1.0-S*f; \n" " float tt=1.0-S*(1.-f); \n" " float newR=0; \n" " float newG=0; \n" " float newB=0; \n" " if((int)hi==0 || (int)hi==6){ \n" " newR=1.0; \n" " newG=tt; \n" " newB=pp; \n" " }else if((int)hi==1){ \n" " newR=qq; \n" " newG=1.0; \n" " newB=pp; \n" " }else if((int)hi==2){ \n" " newR=pp; \n" " newG=1.0; \n" " newB=tt; \n" " }else if((int)hi==3){ \n" " newR=pp; \n" " newG=qq; \n" " newB=1.0; \n" " }else if((int)hi==4){ \n" " newR=tt; \n" " newG=pp; \n" " newB=1.0; \n" " }else if((int)hi==5){ \n" " newR=1.0; \n" " newG=pp; \n" " newB=qq; \n" " } \n" " colorR[id]=(unsigned char)(newR*255.); \n" " colorG[id]=(unsigned char)(newG*255.); \n" " colorB[id]=(unsigned char)(newB*255.); \n" " } \n" "} \n" "__kernel void \n" "calculatePointAssignment(__global float4 const * xyz, __global uchar * mask, __global int const * assignment, \n" " int const radius, int const numFaces, __global uchar * neighbours, __global int * assignmentOut, \n" " int const w, int const h, float const maxDist) \n" "{ \n" " int x = get_global_id(0); \n" " int y = get_global_id(1); \n" " size_t id = x+y*w; \n" " float dist=100000; \n" " int ass=0; \n" " bool assigned=false; \n" " if(mask[id]==0 && assignment[id]==0){ \n" " bool adj[100]; \n" " for(int a=0; a=0 && x+xx=0 && y+yy segmentColorImageRArray; std::vector segmentColorImageGArray; std::vector segmentColorImageBArray; std::vector maskArray; std::vector assignmentArray; //OpenCL utils::CLProgram program; utils::CLKernel kernelSegmentColoring; utils::CLKernel kernelPointAssignment; //OpenCL buffer utils::CLBuffer segmentColorImageRBuffer; utils::CLBuffer segmentColorImageGBuffer; utils::CLBuffer segmentColorImageBBuffer; utils::CLBuffer assignmentBuffer; utils::CLBuffer neighboursBuffer; utils::CLBuffer xyzBuffer; utils::CLBuffer maskBuffer; utils::CLBuffer assignmentOutBuffer; #endif }; SegmenterUtils::SegmenterUtils(Mode mode) : m_data(new Data(mode)) { if(m_data->useCL==true){ #ifdef ICL_HAVE_OPENCL try { m_data->program = utils::CLProgram("gpu", utilsKernel); m_data->kernelSegmentColoring = m_data->program.createKernel("segmentColoring"); m_data->kernelPointAssignment = m_data->program.createKernel("calculatePointAssignment"); m_data->clReady = true; } catch (utils::CLException &err) { //catch openCL errors std::cout<< "ERROR: "<< err.what()<< std::endl; m_data->clReady = false; } #else std::cout << "no openCL parallelization available" << std::endl; m_data->clReady = false; #endif } } SegmenterUtils::~SegmenterUtils() { delete m_data; } core::Img8u SegmenterUtils::createColorImage(core::Img32s &labelImage){ core::Img8u colorImage; if(m_data->useCL==true && m_data->clReady==true){ createColorImageCL(labelImage, colorImage); }else{ createColorImageCPU(labelImage, colorImage); } return colorImage; } core::Img8u SegmenterUtils::createROIMask(core::DataSegment &xyzh, core::Img32f &depthImage, float xMin, float xMax, float yMin, float yMax, float zMin, float zMax){ utils::Size size = depthImage.getSize(); core::Img8u maskImage(size,1,core::formatMatrix); core::Channel8u maskImageC = maskImage[0]; core::Channel32f depthImageC = depthImage[0]; for(int y=0;yxMax || xyzh[i][1]yMax || xyzh[i][2]zMax){ maskImageC(x,y)=1; }else{ maskImageC(x,y)=0; } if(depthImageC(x,y)==2047){ maskImageC(x,y)=1; } } } return maskImage; } core::Img8u SegmenterUtils::createMask(core::Img32f &depthImage){ utils::Size size = depthImage.getSize(); core::Img8u maskImage(size,1,core::formatMatrix); core::Channel8u maskImageC = maskImage[0]; core::Channel32f depthImageC = depthImage[0]; for(int y=0;ystabelizeCounter==0){//first image labelImage.deepCopy(&m_data->lastLabelImage); }else{ core::Channel32s lastLabelImageC = m_data->lastLabelImage[0]; //count number of segments of previous and current label image int countCur=0; int countLast=0; for(int y=0; ycountCur){ countCur=labelImageC(x,y); } if(lastLabelImageC(x,y)>countLast){ countLast=lastLabelImageC(x,y); } } } if(countCur==0 || countLast==0){//no relabeling possible labelImage.deepCopy(&m_data->lastLabelImage); return labelImage; } std::vector curAss = calculateLabelReassignment(countCur, countLast, labelImageC, lastLabelImageC, size); for(int y=0; y0){ stableLabelImageC(x,y)=curAss[labelImageC(x,y)-1]; }else{ stableLabelImageC(x,y)=0; } } } stableLabelImage.deepCopy(&m_data->lastLabelImage);//copy image for next iteration } m_data->stabelizeCounter=1; return stableLabelImage; } math::DynMatrix SegmenterUtils::calculateAdjacencyMatrix(core::DataSegment &xyzh, core::Img32s &labelImage, core::Img8u &maskImage, int radius, float euclideanDistance, int numSurfaces){ math::DynMatrix adjacencyMatrix; if(m_data->useCL==true && m_data->clReady==true){ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCL(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, false); }else{ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCPU(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, false); } return adjacencyMatrix; } void SegmenterUtils::edgePointAssignment(core::DataSegment &xyzh, core::Img32s &labelImage, core::Img8u &maskImage, int radius, float euclideanDistance, int numSurfaces){ math::DynMatrix adjacencyMatrix; if(m_data->useCL==true && m_data->clReady==true){ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCL(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, true); }else{ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCPU(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, true); } } math::DynMatrix SegmenterUtils::edgePointAssignmentAndAdjacencyMatrix(core::DataSegment &xyzh, core::Img32s &labelImage, core::Img8u &maskImage, int radius, float euclideanDistance, int numSurfaces){ math::DynMatrix adjacencyMatrix; if(m_data->useCL==true && m_data->clReady==true){ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCL(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, true); }else{ adjacencyMatrix=edgePointAssignmentAndAdjacencyMatrixCPU(xyzh, labelImage, maskImage, radius, euclideanDistance, numSurfaces, true); } return adjacencyMatrix; } std::vector > SegmenterUtils::extractSegments(core::Img32s &labelImage){ int h=labelImage.getSize().height; int w=labelImage.getSize().width; core::Channel32s labelImageC = labelImage[0]; std::vector > segments; for(int y=0; y(int)segments.size()){ segments.resize(labelImageC(x,y)); } if(labelImageC(x,y)>0){ segments.at(labelImageC(x,y)-1).push_back(x+y*w); } } } return segments; } void SegmenterUtils::relabel(core::Img32s &labelImage, std::vector > &assignment, int maxOldLabel){ std::vector mapping; if(maxOldLabel>0){ mapping.resize(maxOldLabel,0); }else{ int maxLabel=0; for(unsigned int i=0; imaxLabel){ maxLabel=assignment[i][j]+1;//assignment [0..n-1], label [1..n] } } } mapping.resize(maxLabel,0); } for(unsigned int i=0; i0){ labelImageC(x,y)=mapping[labelImageC(x,y)-1]+1; } } } } bool SegmenterUtils::occlusionCheck(core::Img32f &depthImage, utils::Point p1, utils::Point p2, float distanceTolerance, float outlierTolerance){ core::Channel32f depthImageC = depthImage[0]; bool sampleX=false;//over x or y int step=0;//positive or negative float startValue=depthImageC(p1.x,p1.y); float endValue=depthImageC(p2.x,p2.y); float depthGradient, gradient; if(abs(p2.x-p1.x)>abs(p2.y-p1.y)){//sample x init sampleX=true; gradient = (float)(p2.y-p1.y)/(float)(p2.x-p1.x); depthGradient=(endValue-startValue)/(p2.x-p1.x); if(p2.x-p1.x>0){ step=1; }else{ step=-1; } }else{//sample y init sampleX=false; gradient = (float)(p2.x-p1.x)/(float)(p2.y-p1.y); depthGradient=(endValue-startValue)/(p2.y-p1.y); if(p2.y-p1.y>0){ step=1; }else{ step=-1; } } int numReject=0; if(sampleX){//sample x process for(int i=p1.x; (i-p2.x)*step<=0; i+=step){ int newY=(int)round(p1.y+(i-p1.x)*gradient); float realValue = depthImageC(i,newY); float augmentedValue = startValue+(i-p1.x)*depthGradient; float s1 = realValue-augmentedValue;//minus -> real closer than augmented if(s1-distanceTolerance>0 && depthImageC(i,newY)!=2047){//not occluding numReject++; } } if((float)numReject/(float)(abs(p2.x-p1.x)+1)>outlierTolerance/100.){ return false; } }else{//sample y process for(int i=p1.y; (i-p2.y)*step<=0; i+=step){ int newX=(int)round(p1.x+(i-p1.y)*gradient); float realValue = depthImageC(newX,i); float augmentedValue = startValue+(i-p1.y)*depthGradient; float s1 = realValue-augmentedValue; if(s1-distanceTolerance>0 && depthImageC(newX,i)!=2047){ numReject++; } } if((float)numReject/(float)(abs(p2.y-p1.y)+1)>outlierTolerance/100.){ return false; } } return true; } std::vector > SegmenterUtils::createLabelVectors(core::Img32s &labelImage){ utils::Size s = labelImage.getSize(); core::Channel32s labelImageC = labelImage[0]; std::vector > labelVector; for(int y=0; y0){ if(labelImageC(x,y)>(int)labelVector.size()){ labelVector.resize(labelImageC(x,y)); } labelVector[labelImageC(x,y)-1].push_back(id); } } } return labelVector; } void SegmenterUtils::createColorImageCL(core::Img32s &labelImage, core::Img8u &colorImage){ #ifdef ICL_HAVE_OPENCL utils::Size s = labelImage.getSize(); if(s!=m_data->size || m_data->kernelSegmentColoringInitialized==false){//reinit m_data->size = s; int w = s.width; int h = s.height; m_data->segmentColorImageRArray.resize(w*h); m_data->segmentColorImageGArray.resize(w*h); m_data->segmentColorImageBArray.resize(w*h); m_data->segmentColorImageRBuffer = m_data->program.createBuffer("rw", w*h * sizeof(unsigned char)); m_data->segmentColorImageGBuffer = m_data->program.createBuffer("rw", w*h * sizeof(unsigned char)); m_data->segmentColorImageBBuffer = m_data->program.createBuffer("rw", w*h * sizeof(unsigned char)); m_data->assignmentBuffer = m_data->program.createBuffer("r", w*h * sizeof(int)); m_data->kernelSegmentColoringInitialized=true; } try { int w = m_data->size.width; int h = m_data->size.height; m_data->assignmentBuffer.write(labelImage.begin(0),w*h*sizeof(int)); m_data->kernelSegmentColoring.setArgs(m_data->assignmentBuffer, m_data->segmentColorImageRBuffer, m_data->segmentColorImageGBuffer, m_data->segmentColorImageBBuffer); m_data->kernelSegmentColoring.apply(w*h); m_data->segmentColorImageRBuffer.read(&m_data->segmentColorImageRArray[0], w*h * sizeof(unsigned char)); m_data->segmentColorImageGBuffer.read(&m_data->segmentColorImageGArray[0], w*h * sizeof(unsigned char)); m_data->segmentColorImageBBuffer.read(&m_data->segmentColorImageBArray[0], w*h * sizeof(unsigned char)); std::vector data(3); data[0] = m_data->segmentColorImageRArray.data(); data[1] = m_data->segmentColorImageGArray.data(); data[2] = m_data->segmentColorImageBArray.data(); colorImage = core::Img8u(utils::Size(w,h),3,data,false); } catch (utils::CLException &err) { //catch openCL errors std::cout<< "ERROR: "<< err.what() < SegmenterUtils::calculateLabelReassignment(int countCur, int countLast, core::Channel32s &labelImageC, core::Channel32s &lastLabelImageC, utils::Size size){ math::DynMatrix assignmentMatrix(countCur,countLast,0); std::vector lastNum(countLast,0); std::vector curNum(countCur,0); std::vector curAss(countCur,0); std::vector curVal(countCur,0); for(int y=0; y0 && lastLabelImageC(x,y)>0){ assignmentMatrix(labelImageC(x,y)-1, lastLabelImageC(x,y)-1)++;//num match points lastNum[lastLabelImageC(x,y)-1]++;//num segment points curNum[labelImageC(x,y)-1]++;//num segment points } } } for(int i=0; i0){ curScore=(float)assignmentMatrix(i,j)/(float)curNum[i]; lastScore=(float)assignmentMatrix(i,j)/(float)lastNum[j]; compScore=(curScore+lastScore)/2.; } else{ curScore=0; lastScore=0; compScore=0; } if(curVal[i] empties(countCur,true);//find unassigned ids for(int i=0; i SegmenterUtils::edgePointAssignmentAndAdjacencyMatrixCL(core::DataSegment &xyzh, core::Img32s &labelImage, core::Img8u &maskImage, int radius, float euclideanDistance, int numSurfaces, bool pointAssignment){ #ifdef ICL_HAVE_OPENCL utils::Size s = labelImage.getSize(); math::DynMatrix neighbours(numSurfaces,numSurfaces,false); math::DynMatrix neighboursC(numSurfaces,numSurfaces,(unsigned char)0); if(s!=m_data->size || m_data->kernelPointAssignmentInitialized==false){//reinit m_data->size = s; int w = s.width; int h = s.height; m_data->maskArray.resize(w*h); m_data->assignmentArray.resize(w*h); m_data->assignmentBuffer = m_data->program.createBuffer("r", w*h * sizeof(int)); m_data->assignmentOutBuffer = m_data->program.createBuffer("rw", w*h * sizeof(int)); m_data->xyzBuffer = m_data->program.createBuffer("r", w*h * sizeof(Vec)); m_data->maskBuffer = m_data->program.createBuffer("rw", w*h * sizeof(unsigned char)); m_data->kernelPointAssignmentInitialized=true; } try { int w = s.width; int h = s.height; core::Img32s labelImageOut(labelImage.getSize(),1,core::formatMatrix); m_data->assignmentBuffer.write(labelImage.begin(0),w*h*sizeof(int)); m_data->maskBuffer.write(maskImage.begin(0),w*h*sizeof(unsigned char)); m_data->xyzBuffer.write(&xyzh[0][0],w*h*sizeof(Vec));//FixedColVector)); m_data->neighboursBuffer = m_data->program.createBuffer("rw", numSurfaces*numSurfaces * sizeof(unsigned char), &neighboursC[0]); m_data->kernelPointAssignment.setArgs(m_data->xyzBuffer, m_data->maskBuffer, m_data->assignmentBuffer, radius, numSurfaces, m_data->neighboursBuffer, m_data->assignmentOutBuffer, w, h, euclideanDistance); m_data->kernelPointAssignment.apply(w,h); m_data->neighboursBuffer.read(neighboursC.data(), numSurfaces*numSurfaces * sizeof(unsigned char)); for(unsigned int i=0; iassignmentOutBuffer.read(m_data->assignmentArray.data(), w*h * sizeof(int)); labelImage = core::Img32s(utils::Size(w,h),1,std::vector(1,m_data->assignmentArray.data()),false); m_data->maskBuffer.read(m_data->maskArray.data(), w*h * sizeof(unsigned char)); maskImage = core::Img8u(utils::Size(w,h),1,std::vector(1,m_data->maskArray.data()),false); } for(int i=0; i(); #endif } math::DynMatrix SegmenterUtils::edgePointAssignmentAndAdjacencyMatrixCPU(core::DataSegment &xyzh, core::Img32s &labelImage, core::Img8u &maskImage, int radius, float euclideanDistance, int numSurfaces, bool pointAssignment){ utils::Size s = labelImage.getSize(); int w = s.width; int h = s.height; math::DynMatrix neighbours(numSurfaces, numSurfaces, false); core::Img32s labelImageOut(labelImage.getSize(),1,core::formatMatrix); core::Channel32s labelImageC = labelImage[0]; core::Channel32s labelImageOutC = labelImageOut[0]; core::Channel8u maskImageC = maskImage[0]; for (int x=0; x adj(numSurfaces,false); for(int xx=-radius; xx<=radius; xx++) { for(int yy=-radius; yy<=radius; yy++) { if(x+xx>=0 && x+xx=0 && y+yy