Add pick layer to SoPickStyle for ordered on-top picking

include/Inventor/elements/SoElements.h

@@ -64,6 +64,7 @@
 #include <Inventor/elements/SoMaterialBindingElement.h>
 #include <Inventor/elements/SoNormalBindingElement.h>
 #include <Inventor/elements/SoPickStyleElement.h>
+#include <Inventor/elements/SoPickLayerElement.h>
 #include <Inventor/elements/SoSwitchElement.h>
 #include <Inventor/elements/SoTextOutlineEnabledElement.h>
 #include <Inventor/elements/SoTextureCoordinateBindingElement.h>

include/Inventor/elements/SoPickLayerElement.h

@@ -0,0 +1,56 @@
+#ifndef COIN_SOPICKLAYERELEMENT_H
+#define COIN_SOPICKLAYERELEMENT_H
+
+/**************************************************************************\
+ * Copyright (c) Kongsberg Oil & Gas Technologies AS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+\**************************************************************************/
+
+#include <Inventor/elements/SoInt32Element.h>
+
+class COIN_DLL_API SoPickLayerElement : public SoInt32Element {
+  typedef SoInt32Element inherited;
+
+  SO_ELEMENT_HEADER(SoPickLayerElement);
+public:
+  static void initClass(void);
+protected:
+  virtual ~SoPickLayerElement();
+
+public:
+  virtual void init(SoState * state);
+  static void set(SoState * const state, SoNode * const node,
+                  const int32_t layer);
+  static void set(SoState * const state, const int32_t layer);
+  static int32_t get(SoState * const state);
+  static int32_t getDefault();
+};
+
+#endif // !COIN_SOPICKLAYERELEMENT_H

include/Inventor/nodes/SoPickStyle.h

@@ -35,6 +35,7 @@
 
 #include <Inventor/nodes/SoSubNode.h>
 #include <Inventor/fields/SoSFEnum.h>
+#include <Inventor/fields/SoSFInt32.h>
 #include <Inventor/elements/SoPickStyleElement.h>
 
 class COIN_DLL_API SoPickStyle : public SoNode {
@@ -56,6 +57,7 @@ class COIN_DLL_API SoPickStyle : public SoNode {
   };
 
   SoSFEnum style;
+  SoSFInt32 layer;
 
   virtual void doAction(SoAction * action);
   virtual void callback(SoCallbackAction * action);

src/actions/SoRayPickAction.cpp

@@ -130,6 +130,7 @@
 #include <Inventor/actions/SoRayPickAction.h>
 
 #include <cfloat>
+#include <cstdint>
 
 #include <Inventor/SbLine.h>
 #include <Inventor/SoPickedPoint.h>
@@ -139,6 +140,7 @@
 #include <Inventor/elements/SoTextureOverrideElement.h>
 #include <Inventor/elements/SoPickRayElement.h>
 #include <Inventor/elements/SoPickStyleElement.h>
+#include <Inventor/elements/SoPickLayerElement.h>
 #include <Inventor/elements/SoShapeHintsElement.h>
 #include <Inventor/elements/SoViewVolumeElement.h>
 #include <Inventor/elements/SoViewportRegionElement.h>
@@ -169,7 +171,7 @@
 
 class SoRayPickActionP {
 public:
-  SoRayPickActionP(void) : owner(NULL) { }
+  SoRayPickActionP(void) : pickstylelayer(0), owner(NULL) { }
 
   // Hidden private methods.
 
@@ -209,6 +211,11 @@ class SoRayPickActionP {
 
   SoPickedPointList pickedpointlist;
   SbList <double> ppdistance;
+  SbList <int32_t> pplayer;
+
+  // the pick layer for the shape currently being intersected (set in
+  // setPickStyleFlags); on-top picks ignore this and always sort frontmost
+  int32_t pickstylelayer;
 
   unsigned int flags;
   SbBool objectspacevalid; // FIXME: why not a flag?
@@ -437,24 +444,36 @@ SoRayPickAction::getPickedPointList(void) const
   if (!PRIVATE(this)->isFlagSet(SoRayPickActionP::PPLIST_IS_SORTED) && n > 1) {
     SoPickedPoint ** pparray = reinterpret_cast<SoPickedPoint **>(PRIVATE(this)->pickedpointlist.getArrayPtr());
     double * darray = const_cast<double*>(PRIVATE(this)->ppdistance.getArrayPtr());
+    int32_t * larray = const_cast<int32_t*>(PRIVATE(this)->pplayer.getArrayPtr());
 
     int i, j, distance;
     SoPickedPoint * pptmp;
     double dtmp;
+    int32_t ltmp;
+
+    // a point sorts in front of another when it is in a higher layer, or in the
+    // same layer but closer; on-top picks carry the maximum layer and thus stay
+    // frontmost regardless of depth
+    auto sortsBefore = [](int32_t layerA, double distA, int32_t layerB, double distB) {
+      return layerA > layerB || (layerA == layerB && distA < distB);
+    };
 
     // shell sort algorithm (O(nlog(n))
     for (distance = 1; distance <= n/9; distance = 3*distance + 1) ;
     for (; distance > 0; distance /= 3) {
       for (i = distance; i < n; i++) {
         dtmp = darray[i];
+        ltmp = larray[i];
         pptmp = pparray[i];
         j = i;
-        while (j >= distance && darray[j-distance] > dtmp) {
+        while (j >= distance && sortsBefore(ltmp, dtmp, larray[j-distance], darray[j-distance])) {
           darray[j] = darray[j-distance];
+          larray[j] = larray[j-distance];
           pparray[j] = pparray[j-distance];
           j -= distance;
         }
         darray[j] = dtmp;
+        larray[j] = ltmp;
         pparray[j] = pptmp;
       }
     }
@@ -1018,20 +1037,30 @@ SoRayPickAction::addIntersection(const SbVec3f & objectspacepoint_in, SbBool fro
   PRIVATE(this)->obj2world.multVecMatrix(objectspacepoint, worldpoint);
   double dist = PRIVATE(this)->isFlagSet(SoRayPickActionP::PUSH_PICK_TO_FRONT) ?
     0.0 : PRIVATE(this)->nearplane.getDistance(worldpoint);
+  // on-top picks ignore their layer and always sort frontmost
+  int32_t layer = PRIVATE(this)->isFlagSet(SoRayPickActionP::PUSH_PICK_TO_FRONT) ?
+    INT32_MAX : PRIVATE(this)->pickstylelayer;
 
   if (!PRIVATE(this)->isFlagSet(SoRayPickActionP::PICK_ALL) && PRIVATE(this)->pickedpointlist.getLength()) {
-    // got to test if new candidate is closer than old one
-    if (dist >= PRIVATE(this)->ppdistance[0]) return NULL; // farther
+    // got to test if new candidate sorts in front of the old one: higher layer
+    // wins, and within a layer the closer point wins
+    const int32_t prevlayer = PRIVATE(this)->pplayer[0];
+    const double prevdist = PRIVATE(this)->ppdistance[0];
+    if (layer < prevlayer || (layer == prevlayer && dist >= prevdist)) {
+      return NULL; // behind
+    }
     // remove old point
     PRIVATE(this)->pickedpointlist.truncate(0);
     PRIVATE(this)->ppdistance.truncate(0);
+    PRIVATE(this)->pplayer.truncate(0);
   }
 
   // create the new picked point
   SoPickedPoint * pp = new SoPickedPoint(this->getCurPath(),
                                          this->state, objectspacepoint_in);
   PRIVATE(this)->pickedpointlist.append(pp);
   PRIVATE(this)->ppdistance.append(dist);
+  PRIVATE(this)->pplayer.append(layer);
   PRIVATE(this)->clearFlag(SoRayPickActionP::PPLIST_IS_SORTED);
   return pp;
 }
@@ -1091,6 +1120,7 @@ SoRayPickActionP::cleanupPickedPoints(void)
 {
   this->pickedpointlist.truncate(0); // this will delete all SoPickedPoint instances in the list
   this->ppdistance.truncate(0);
+  this->pplayer.truncate(0);
   this->clearFlag(PPLIST_IS_SORTED);
 }
 
@@ -1153,8 +1183,11 @@ void
 SoRayPickActionP::setPickStyleFlags(SoState * state)
 {
   assert(state->isElementEnabled(SoPickStyleElement::getClassStackIndex()));
+  assert(state->isElementEnabled(SoPickLayerElement::getClassStackIndex()));
   assert(state->isElementEnabled(SoShapeHintsElement::getClassStackIndex()));
 
+  this->pickstylelayer = SoPickLayerElement::get(state);
+
   SoPickStyleElement::Style style = SoPickStyleElement::get(state);
   SoShapeHintsElement::ShapeType type = SoShapeHintsElement::getShapeType(state);
   SoShapeHintsElement::VertexOrdering ordering = SoShapeHintsElement::getVertexOrdering(state);

src/elements/CMakeLists.txt

@@ -52,6 +52,7 @@ set(COIN_ELEMENTS_FILES
 	SoOverrideElement.cpp
 	SoPickRayElement.cpp
 	SoPickStyleElement.cpp
+	SoPickLayerElement.cpp
 	SoPointSizeElement.cpp
 	SoPolygonOffsetElement.cpp
 	SoProfileCoordinateElement.cpp

src/elements/SoElement.cpp

@@ -400,6 +400,7 @@ SoElement::initElements(void)
   SoMaterialBindingElement::initClass();
   SoNormalBindingElement::initClass();
   SoPickStyleElement::initClass();
+  SoPickLayerElement::initClass();
   SoSwitchElement::initClass();
   SoTextOutlineEnabledElement::initClass();
   SoTextureCoordinateBindingElement::initClass();

src/elements/SoPickLayerElement.cpp

@@ -0,0 +1,114 @@
+/**************************************************************************\
+ * Copyright (c) Kongsberg Oil & Gas Technologies AS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+\**************************************************************************/
+
+/*!
+  \class SoPickLayerElement Inventor/elements/SoPickLayerElement.h
+  \brief The SoPickLayerElement holds the current pick layer.
+
+  \ingroup coin_elements
+
+  The pick layer orders picked points across the scene graph. Picked points
+  in a higher layer sort in front of those in a lower layer, independent of
+  their depth. It is paired with SoPickStyleElement: on-top pick styles ignore
+  the layer and always sort frontmost. The default layer is 0.
+*/
+
+#include <Inventor/elements/SoPickLayerElement.h>
+
+SO_ELEMENT_SOURCE(SoPickLayerElement);
+
+/*!
+  \copydetails SoElement::initClass(void)
+*/
+
+void
+SoPickLayerElement::initClass(void)
+{
+  SO_ELEMENT_INIT_CLASS(SoPickLayerElement, inherited);
+}
+
+/*!
+  Destructor.
+*/
+
+SoPickLayerElement::~SoPickLayerElement()
+{
+}
+
+//! Sets the current pick layer.
+
+void
+SoPickLayerElement::set(SoState * const state,
+                        SoNode * const node,
+                        const int32_t layer)
+{
+  SoInt32Element::set(classStackIndex, state, node, layer);
+}
+
+/*!
+  Initializes the element to its default value. The default
+  value for the pick layer is 0.
+*/
+
+void
+SoPickLayerElement::init(SoState * state)
+{
+  inherited::init(state);
+  this->data = getDefault();
+}
+
+//! Sets the current pick layer.
+
+//$ EXPORT INLINE
+void
+SoPickLayerElement::set(SoState * const state, const int32_t layer)
+{
+  set(state, NULL, layer);
+}
+
+//! Returns the current pick layer.
+
+//$ EXPORT INLINE
+int32_t
+SoPickLayerElement::get(SoState * const state)
+{
+  return SoInt32Element::get(classStackIndex, state);
+}
+
+//! Returns the default pick layer (0).
+
+//$ EXPORT INLINE
+int32_t
+SoPickLayerElement::getDefault()
+{
+  return 0;
+}

src/nodes/SoPickStyle.cpp

@@ -59,6 +59,7 @@
 #include <Inventor/actions/SoPickAction.h>
 
 #include <Inventor/elements/SoOverrideElement.h>
+#include <Inventor/elements/SoPickLayerElement.h>
 #include <Inventor/actions/SoCallbackAction.h>
 
 #include "nodes/SoSubNodeP.h"
@@ -130,6 +131,16 @@
   in the scene graph. Default value is SoPickStyle::SHAPE.
 */
 
+/*!
+  \var SoSFInt32 SoPickStyle::layer
+
+  The pick layer for subsequent shapes in the scene graph. Picked points
+  in a higher layer sort in front of those in a lower layer, regardless of
+  their depth; within a layer, the usual depth order applies. The on-top
+  pick styles ignore the layer and always sort frontmost. Default value
+  is 0.
+*/
+
 // *************************************************************************
 
 SO_NODE_SOURCE(SoPickStyle);
@@ -142,6 +153,7 @@ SoPickStyle::SoPickStyle(void)
   SO_NODE_INTERNAL_CONSTRUCTOR(SoPickStyle);
 
   SO_NODE_ADD_FIELD(style, (SoPickStyle::SHAPE));
+  SO_NODE_ADD_FIELD(layer, (0));
 
   SO_NODE_DEFINE_ENUM_VALUE(Style, SHAPE);
   SO_NODE_DEFINE_ENUM_VALUE(Style, BOUNDING_BOX);
@@ -170,6 +182,9 @@ SoPickStyle::initClass(void)
 
   SO_ENABLE(SoCallbackAction, SoPickStyleElement);
   SO_ENABLE(SoPickAction, SoPickStyleElement);
+
+  SO_ENABLE(SoCallbackAction, SoPickLayerElement);
+  SO_ENABLE(SoPickAction, SoPickLayerElement);
 }
 
 void
@@ -183,6 +198,9 @@ SoPickStyle::doAction(SoAction * action)
       SoOverrideElement::setPickStyleOverride(action->getState(), this, TRUE);
     }
   }
+  if (!this->layer.isIgnored()) {
+    SoPickLayerElement::set(action->getState(), this, this->layer.getValue());
+  }
 }
 
 void