Bootstrap with prior depth

Author: Fabien Servant

meshroom/aliceVision/SfmBootstrapping.py

@@ -18,6 +18,13 @@ class SfMBootStrapping(desc.AVCommandLineNode):
             description="SfMData file.",
             value="",
         ),
+        desc.ChoiceParam(
+            name="method",
+            label="Method",
+            description="Bootstrapping method: classic (epipolar geometry), mesh (3D mesh constraints), or depth (depth map information).",
+            values=["classic", "mesh", "depth"],
+            value="classic",
+        ),
         desc.File(
             name="tracksFilename",
             label="Tracks File",
@@ -29,6 +36,7 @@ class SfMBootStrapping(desc.AVCommandLineNode):
             label="Mesh File",
             description="Mesh file (*.obj).",
             value="",
+            enabled=lambda node: node.method.value == "mesh"
         ),
         desc.File(
             name="pairs",

src/aliceVision/sfm/CMakeLists.txt

@@ -27,6 +27,7 @@ set(sfm_files_headers
     pipeline/bootstrapping/EstimateAngle.hpp
     pipeline/bootstrapping/PairsScoring.hpp
     pipeline/bootstrapping/Bootstrap.hpp
+    pipeline/bootstrapping/TracksDepths.hpp
     pipeline/expanding/SfmTriangulation.hpp
     pipeline/expanding/SfmResection.hpp
     pipeline/expanding/SfmBundle.hpp
@@ -74,6 +75,7 @@ set(sfm_files_sources
     pipeline/bootstrapping/EstimateAngle.cpp
     pipeline/bootstrapping/PairsScoring.cpp
     pipeline/bootstrapping/Bootstrap.cpp
+    pipeline/bootstrapping/TracksDepths.cpp
     pipeline/expanding/SfmTriangulation.cpp
     pipeline/expanding/SfmResection.cpp
     pipeline/expanding/SfmBundle.cpp

src/aliceVision/sfm/pipeline/bootstrapping/Bootstrap.cpp

@@ -10,6 +10,7 @@
 #include <aliceVision/multiview/triangulation/triangulationDLT.hpp>
 #include <aliceVision/sfm/pipeline/expanding/SfmResection.hpp>
 #include <aliceVision/sfm/pipeline/expanding/LocalizationValidationPolicyLegacy.hpp>
+#include <aliceVision/sfm/pipeline/bootstrapping/TracksDepths.hpp>
 #include <vector>
 #include <random>
 
@@ -174,5 +175,102 @@ bool bootstrapMesh(sfmData::SfMData & sfmData,
     return true;
 }
 
+bool bootstrapDepth(sfmData::SfMData & sfmData, 
+                    const IndexT referenceViewId,
+                    const IndexT nextViewId,
+                    const track::TracksMap& tracksMap, 
+                    const track::TracksPerView & tracksPerView)
+{
+    std::mt19937 randomNumberGenerator;
+
+    const sfmData::View & viewReference = sfmData.getView(referenceViewId);
+    const sfmData::View & viewNext = sfmData.getView(nextViewId);
+
+    camera::IntrinsicBase::sptr camReference = sfmData.getIntrinsicSharedPtr(viewReference.getIntrinsicId());
+    camera::IntrinsicBase::sptr camNext = sfmData.getIntrinsicSharedPtr(viewNext.getIntrinsicId());
+
+    sfmData::CameraPose & poseReference = sfmData.getPoses()[viewReference.getPoseId()];
+    sfmData::CameraPose & poseNext = sfmData.getPoses()[viewNext.getPoseId()];
+
+
+    sfmData::SfMData miniSfm;
+    if (!buildSfmDataFromDepthMap(miniSfm, sfmData, tracksMap, tracksPerView, referenceViewId))
+    {
+        return false;
+    }
+
+    //Compute resection for selected view
+    sfm::LocalizationValidationPolicy::uptr resectionValidationPolicy = std::make_unique<sfm::LocalizationValidationPolicyLegacy>();
+
+    sfm::SfmResection resection;
+    resection.setMaxIterations(50000);
+    resection.setResectionMaxError(std::numeric_limits<double>::infinity());
+    resection.setValidationPolicy(resectionValidationPolicy);
+
+    Eigen::Matrix4d pose;
+    double newThreshold;
+    size_t inliersCount;
+
+    if (!resection.processView(miniSfm, 
+                            tracksMap, tracksPerView, 
+                            randomNumberGenerator,
+                            nextViewId, pose, newThreshold, inliersCount))
+    {
+        return false;
+    }
+
+
+    geometry::Pose3 pose3(pose);
+    poseNext.setTransform(pose3);
+
+    const auto & landmarks = miniSfm.getLandmarks();
+    auto & outLandmarks = sfmData.getLandmarks();
+
+    for (const auto & [landmarkId, landmark] : landmarks)
+    {
+        //Retrieve track object
+        const auto & track = tracksMap.at(landmarkId);
+
+        const track::TrackItem & itemReference = track.featPerView.at(referenceViewId);
+
+        //Maybe this track is not observed in the next view
+        if (track.featPerView.find(nextViewId) == track.featPerView.end())
+        {
+            continue;
+        }
+
+        //Compute error
+        const track::TrackItem & item = track.featPerView.at(nextViewId);
+        const Vec2 pt = item.coords;
+        const Vec2 estpt = camNext->transformProject(pose3, landmark.X.homogeneous(), true);
+        double err = (pt - estpt).norm();
+
+        //If error is ok, then we add it to the sfmData
+        if (err <= newThreshold)
+        {
+            sfmData::Observation obs;
+            obs.setFeatureId(item.featureId);
+            obs.setScale(item.scale);
+            obs.setCoordinates(item.coords);
+
+            sfmData::Observation obsReference;
+            obsReference.setFeatureId(itemReference.featureId);
+            obsReference.setScale(itemReference.scale);
+            obsReference.setCoordinates(itemReference.coords);
+
+            //Add landmark to sfmData
+            outLandmarks[landmarkId] = landmark;
+            outLandmarks[landmarkId].setParallaxRobust(true);
+
+            //Add observation to landmark
+            sfmData::Observations & observations = outLandmarks[landmarkId].getObservations();
+            observations[referenceViewId] = obsReference;
+            observations[nextViewId] = obs;
+        }
+    }
+
+    return true;
+}
+
 }
 }

src/aliceVision/sfm/pipeline/bootstrapping/Bootstrap.hpp

@@ -46,5 +46,20 @@ bool bootstrapMesh(sfmData::SfMData & sfmData,
                     const track::TracksMap& tracksMap, 
                     const track::TracksPerView & tracksPerView);
 
+/**
+ * @brief Create a minimal SfmData with poses and landmarks for two views
+ * @param sfmData the input sfmData which contains camera information
+ * @param referenceViewId the reference view id
+ * @param otherViewId the other view id
+ * @param tracksMap the input map of tracks
+ * @param tracksPerView tracks grouped by views
+ * @return true
+*/
+bool bootstrapDepth(sfmData::SfMData & sfmData, 
+                    const IndexT referenceViewId,
+                    const IndexT otherViewId,
+                    const track::TracksMap& tracksMap, 
+                    const track::TracksPerView & tracksPerView);
+
 }
 }

src/aliceVision/sfm/pipeline/bootstrapping/PairsScoring.cpp

@@ -8,6 +8,9 @@
 #include <aliceVision/sfm/pipeline/bootstrapping/EstimateAngle.hpp>
 #include <aliceVision/sfm/pipeline/expanding/ExpansionPolicyLegacy.hpp>
 #include <aliceVision/multiview/triangulation/triangulationDLT.hpp>
+#include <aliceVision/sfm/pipeline/bootstrapping/TracksDepths.hpp>
+#include <aliceVision/sfm/pipeline/expanding/SfmResection.hpp>
+#include <aliceVision/sfm/pipeline/expanding/LocalizationValidationPolicyLegacy.hpp>
 
 namespace aliceVision {
 namespace sfm {
@@ -116,5 +119,110 @@ IndexT findBestPair(const sfmData::SfMData & sfmData,
     return bestPair;
 }
 
+sfm::ReconstructedPair findBestPairFromTrackDepths(const sfmData::SfMData & sfmData, 
+                                    const std::vector<sfm::ReconstructedPair> & pairs,
+                                    const track::TracksMap& tracksMap, 
+                                    const track::TracksPerView & tracksPerView,
+                                    std::mt19937 & randomNumberGenerator)
+{
+    //Create set of unique view ids
+    std::set<IndexT> views;
+    for (IndexT pairId = 0; pairId < pairs.size(); pairId++)
+    {
+        views.insert(pairs[pairId].reference);
+        views.insert(pairs[pairId].next);
+    }
+
+    sfm::ReconstructedPair bestPair;
+    bestPair.reference = UndefinedIndexT;
+    size_t bestCount = 0;
+
+    sfm::LocalizationValidationPolicy::uptr resectionValidationPolicy = std::make_unique<sfm::LocalizationValidationPolicyLegacy>();
+
+    sfm::SfmResection resection;
+    resection.setMaxIterations(1024);
+    resection.setResectionMaxError(std::numeric_limits<double>::infinity());
+    resection.setValidationPolicy(resectionValidationPolicy);
+
+    //Loop over all views relatively located to another view
+    for (const auto & idView: views)
+    {
+        //Build a local sfm where the landmarks are the points on the depthmap
+        sfmData::SfMData miniSfm;
+        if (!buildSfmDataFromDepthMap(miniSfm, sfmData, tracksMap, tracksPerView, idView))
+        {
+            continue;
+        }
+        
+        sfm::ReconstructedPair bestPairLocal;
+        size_t maxInliers = 0;
+        size_t totalInliers = 0;
+
+        //Loop over all other views which are linked to this view
+        for (IndexT pairId = 0; pairId < pairs.size(); pairId++)
+        {
+            const auto & pairSource = pairs[pairId];
+            sfm::ReconstructedPair pair;
+            pair.reference = idView;
+
+
+            // Create the correct pair with order
+            if (pairSource.reference == idView)
+            {
+                pair.next = pairSource.next;
+            }
+            else if (pairSource.next == idView)
+            {
+                pair.next = pairSource.reference;
+            }
+            else
+            {
+                continue;
+            }
+
+            //Make sure the local sfm has this view
+            if (miniSfm.getViews().find(pair.next) == miniSfm.getViews().end())
+            {
+                continue;
+            }
+
+            //Try to locate this view in the local sfm
+            Eigen::Matrix4d pose;
+            double newThreshold;
+            size_t inliersCount;
+            if (!resection.processView(miniSfm, 
+                                    tracksMap, 
+                                    tracksPerView, 
+                                    randomNumberGenerator,
+                                    pair.next, 
+                                    pose, newThreshold, inliersCount))
+            {
+                continue;
+            }    
+
+            //Keep the best second view
+            pair.pose = geometry::Pose3(pose);
+            pair.score = inliersCount;
+            if (inliersCount > maxInliers)
+            {
+                maxInliers = inliersCount;
+                bestPairLocal = pair;
+            }
+
+            totalInliers += inliersCount;
+        }
+
+        //Keep the best pair
+        if (totalInliers > bestCount)
+        {
+            bestPair = bestPairLocal;
+            bestCount = totalInliers;
+        }
+    }
+
+
+    return bestPair;
+}
+
 }
 }

src/aliceVision/sfm/pipeline/bootstrapping/PairsScoring.hpp

@@ -37,5 +37,20 @@ IndexT findBestPair(const sfmData::SfMData & sfmData,
                     double softMinAngle,
                     double maxAngle);
 
+/**
+* @brief Get best pair from track Depths with highest score
+* @param sfmData the input sfmData which contains camera information
+* @param pairs the input list of reconstructed pairs
+* @param tracksMap the input map of tracks
+* @param tracksPerView tracks grouped by views
+* @param randomNumberGenerator the random number generator used for drawing numbers
+* @return The best pair (pair.reference is UndefinedIndexT if nothing found)
+*/
+sfm::ReconstructedPair findBestPairFromTrackDepths(const sfmData::SfMData & sfmData, 
+                            const std::vector<sfm::ReconstructedPair> & pairs,
+                            const track::TracksMap& tracksMap, 
+                            const track::TracksPerView & tracksPerView,
+                            std::mt19937 & randomNumberGenerator);
+
 }
 }

src/aliceVision/sfm/pipeline/bootstrapping/TracksDepths.cpp

@@ -0,0 +1,82 @@
+// This file is part of the AliceVision project.
+// Copyright (c) 2025 AliceVision contributors.
+// This Source Code Form is subject to the terms of the Mozilla Public License,
+// v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#include <aliceVision/sfm/pipeline/bootstrapping/TracksDepths.hpp>
+
+namespace aliceVision {
+namespace sfm {
+
+bool buildSfmDataFromDepthMap(sfmData::SfMData & output,
+                            const sfmData::SfMData & sfmData, 
+                            const track::TracksMap& tracksMap, 
+                            const track::TracksPerView & tracksPerView, 
+                            IndexT viewId)
+{
+    output.clear();
+
+    const sfmData::View & view = sfmData.getView(viewId);
+    const camera::IntrinsicBase & intrinsic = sfmData.getIntrinsic(view.getIntrinsicId());
+
+    if (tracksPerView.find(viewId) == tracksPerView.end())
+    {
+        return false;
+    }
+
+    sfmData::Landmarks & landmarks = output.getLandmarks();
+
+    //Copy all intrinsics, because it's light.
+    for (const auto & [intrinsicId, intrinsic] : sfmData.getIntrinsics())
+    {
+        output.getIntrinsics().insert(
+            std::make_pair(intrinsicId, 
+                camera::IntrinsicBase::sptr(intrinsic->clone()))
+            );
+    }
+
+    std::set<IndexT> usedViewIds;
+    const auto & trackIds = tracksPerView.at(viewId);
+    for (const auto & trackId : trackIds)
+    {
+        const auto & track = tracksMap.at(trackId);
+        const auto & feat = track.featPerView.at(viewId);
+        const double & Z = feat.depth;
+
+        if (Z <= 0.0)
+        {
+            continue;
+        }
+        
+        const Vec2 meters = intrinsic.removeDistortion(intrinsic.ima2cam(feat.coords.cast<double>()));
+        
+        sfmData::Landmark & landmark = landmarks[trackId];
+        landmark.X.x() = meters.x() * Z;
+        landmark.X.y() = meters.y() * Z;
+        landmark.X.z() = Z;
+
+        landmark.descType = track.descType;
+
+        for (const auto & [otherViewId, otherFeat] : track.featPerView)
+        {
+            usedViewIds.insert(otherViewId);
+        }
+    }
+
+    // Copy only used views
+    for (const auto & usedViewId : usedViewIds)
+    {
+        const auto & iview = sfmData.getViewSharedPtr(usedViewId);
+
+        output.getViews().insert(
+            std::make_pair(usedViewId, 
+                sfmData::View::sptr(iview->clone()))
+            );
+    }
+
+    return true;
+}
+
+}
+}