Infrastructure for reporting progress

This commit adds two classes:
* ProgressTracker
* ProgressScope

The first is for users to implement, and to instantiate when they desire
to get informed about the overall progress.

The second is to be added to all functions that may take a considerable
amount of time, such that they can report back how far along they are.

These are much more convenient to use than the existing ProgressHandler.
ProgressScope is designed such that it only requires "local knowledge"
about upcoming and finished work. Scopes are nested and combined to
form the final global progress.

The active ProgressTracker is stored in a thread_local pointer.
This is a consicius decision since in assimp there is often no 'context'
passed through. The ProgressTracker may be needed anywhere, and it would
be tedious and a huge change to pass it through to every function.
Therefore, using a thread_local variable makes it accessible everywhere,
without a major interface change. Since assimmp is single-threaded,
but may be run in parallel on multiple threads, a thread_local is a
good trade-off, in my opinion.

This change only adds few uses of ProgressScope, to generally show how
it would be used. Also for our use cases these where the most pressing
places to add progress reporting, so this already covers loading from FBX
files pretty well.

code/AssetLib/FBX/FBXConverter.cpp

@@ -62,6 +62,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <assimp/StringUtils.h>
 #include <assimp/commonMetaData.h>
 
+#include <assimp/ProgressTracker.h>
 #include <stdlib.h>
 #include <cstdint>
 #include <iomanip>
@@ -94,24 +95,50 @@ FBXConverter::FBXConverter(aiScene *out, const Document &doc, bool removeEmptyBo
         doc(doc),
         mRemoveEmptyBones(removeEmptyBones) {
 
+    Assimp::ProgressScope progScope("FBX Converter");
+    {
+        progScope.AddStep();        // Converting Animations
+
+        if (doc.Settings().readTextures)
+            progScope.AddStep();    // Converting Embedded Textures
+
+        progScope.AddStep(2);       // Converting Root Node
+
+        if (doc.Settings().readAllMaterials)
+            progScope.AddStep();    // Reading Materials
+
+        progScope.AddStep();        // Transferring Data to Scene
+    }
 
     // animations need to be converted first since this will
     // populate the node_anim_chain_bits map, which is needed
     // to determine which nodes need to be generated.
+    progScope.StartStep("Converting Animations");
     ConvertAnimations();
+
     // Embedded textures in FBX could be connected to nothing but to itself,
     // for instance Texture -> Video connection only but not to the main graph,
     // The idea here is to traverse all objects to find these Textures and convert them,
     // so later during material conversion it will find converted texture in the textures_converted array.
     if (doc.Settings().readTextures) {
+        progScope.StartStep("Converting Embedded Textures");
         ConvertOrphanedEmbeddedTextures();
     }
+
+    progScope.StartStep("Converting Root Node");
     ConvertRootNode();
 
     if (doc.Settings().readAllMaterials) {
+        progScope.StartStep("Reading Materials");
+
+        Assimp::ProgressScope materialProgScope("Read All Materials");
+        materialProgScope.AddSteps(doc.Objects().size());
+
         // unfortunately this means we have to evaluate all objects
         for (const ObjectMap::value_type &v : doc.Objects()) {
 
+            materialProgScope.StartStep("Converting Material");
+
             const Object *ob = v.second->Get();
             if (!ob) {
                 continue;
@@ -127,6 +154,8 @@ FBXConverter::FBXConverter(aiScene *out, const Document &doc, bool removeEmptyBo
         }
     }
 
+    progScope.StartStep("Transferring Data to Scene");
+
     ConvertGlobalSettings();
     TransferDataToScene();
 
@@ -207,7 +236,13 @@ void FBXConverter::ConvertNodes(uint64_t id, aiNode *parent, aiNode *root_node)
     std::vector<PotentialNode> nodes_chain;
     std::vector<PotentialNode> post_nodes_chain;
 
+    Assimp::ProgressScope progScope("Convert FBX Nodes");
+    progScope.AddSteps(conns.size());
+
     for (const Connection *con : conns) {
+
+        progScope.StartStep("Converting Node");
+
         // ignore object-property links
         if (con->PropertyName().length()) {
             // really important we document why this is ignored.

code/AssetLib/FBX/FBXImporter.cpp

@@ -54,6 +54,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "FBXUtil.h"
 
 #include <assimp/MemoryIOWrapper.h>
+#include <assimp/ProgressTracker.h>
 #include <assimp/StreamReader.h>
 #include <assimp/importerdesc.h>
 #include <assimp/Importer.hpp>
@@ -137,6 +138,12 @@ void FBXImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 	}
 
     ASSIMP_LOG_DEBUG("Reading FBX file");
+    Assimp::ProgressScope progScope("Read FBX File");
+    progScope.AddStep(5);  // Reading FBX File Content
+    progScope.AddStep();   // Tokenizing FBX
+    progScope.AddStep(24); // Converting FBX to aiScene
+
+    progScope.StartStep("Reading FBX File Content");
 
 	// read entire file into memory - no streaming for this, fbx
 	// files can grow large, but the assimp output data structure
@@ -154,6 +161,8 @@ void FBXImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 	TokenList tokens;
     Assimp::StackAllocator tempAllocator;
     try {
+        progScope.StartStep("Tokenizing FBX");
+
 		bool is_binary = false;
 		if (!strncmp(begin, "Kaydara FBX Binary", 18)) {
 			is_binary = true;
@@ -162,6 +171,8 @@ void FBXImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
             Tokenize(tokens, begin, tempAllocator);
 		}
 
+        progScope.StartStep("Converting FBX to aiScene");
+
 		// use this information to construct a very rudimentary
 		// parse-tree representing the FBX scope structure
         Parser parser(tokens, tempAllocator, is_binary);

code/CMakeLists.txt

@@ -79,6 +79,7 @@ SET( PUBLIC_HEADERS
   ${HEADER_PATH}/mesh.h
   ${HEADER_PATH}/ObjMaterial.h
   ${HEADER_PATH}/pbrmaterial.h
+  ${HEADER_PATH}/ProgressTracker.h
   ${HEADER_PATH}/GltfMaterial.h
   ${HEADER_PATH}/postprocess.h
   ${HEADER_PATH}/quaternion.h
@@ -211,6 +212,7 @@ SET( Common_SRCS
   Common/AssertHandler.cpp
   Common/Exceptional.cpp
   Common/Base64.cpp
+  Common/ProgressTracker.cpp
 )
 SOURCE_GROUP(Common FILES ${Common_SRCS})
 

code/Common/Importer.cpp

@@ -85,6 +85,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <assimp/DefaultIOStream.h>
 #include <assimp/DefaultIOSystem.h>
+#include <assimp/ProgressTracker.h>
 
 #ifndef ASSIMP_BUILD_NO_VALIDATEDS_PROCESS
 #   include "PostProcessing/ValidateDataStructure.h"
@@ -596,6 +597,12 @@ const aiScene* Importer::ReadFile( const char* _pFile, unsigned int pFlags) {
     // ImportErrorException's are throw by ourselves and caught elsewhere.
     //-----------------------------------------------------------------------
 
+    Assimp::ProgressScope progScope("Read File");
+    progScope.AddStep(35);  // Importing File
+    progScope.AddStep();    // Post Processing Scene
+
+    progScope.StartStep("Importing File");
+
     WriteLogOpening(pFile);
 
 #ifdef ASSIMP_CATCH_GLOBAL_EXCEPTIONS
@@ -713,6 +720,8 @@ const aiScene* Importer::ReadFile( const char* _pFile, unsigned int pFlags) {
             profiler->EndRegion("import");
         }
 
+        progScope.StartStep("Post Processing Scene");
+
         SetPropertyString("sourceFilePath", pFile);
 
         // If successful, apply all active post processing steps to the imported data

code/Common/ProgressTracker.cpp

@@ -0,0 +1,198 @@
+/*
+---------------------------------------------------------------------------
+Open Asset Import Library (assimp)
+---------------------------------------------------------------------------
+
+Copyright (c) 2006-2022, assimp team
+
+
+
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the following
+conditions are met:
+
+* Redistributions of source code must retain the above
+copyright notice, this list of conditions and the
+following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the
+following disclaimer in the documentation and/or other
+materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+contributors may be used to endorse or promote products
+derived from this software without specific prior
+written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+---------------------------------------------------------------------------
+*/
+
+#include <assimp/ProgressTracker.h>
+
+#include <cassert>
+
+namespace Assimp {
+
+thread_local ProgressTracker *g_progressTracker = nullptr;
+
+ProgressTracker::ProgressTracker() = default;
+
+ProgressTracker::~ProgressTracker() {
+    if (g_progressTracker == this) {
+        g_progressTracker = nullptr;
+    }
+
+    assert(currentScope == nullptr && "ProgressScope still exists during ProgressTracker destruction");
+}
+
+void ProgressTracker::SetThreadLocalProgressTracker(ProgressTracker *tracker) {
+    g_progressTracker = tracker;
+}
+
+ProgressScope::ProgressScope(const char *scopeName) :
+        scopeName(scopeName) {
+
+    progressTracker = g_progressTracker;
+
+    if (progressTracker) {
+        progressTracker->Lock();
+        parentScope = progressTracker->currentScope;
+        progressTracker->currentScope = this;
+        progressTracker->Unlock();
+    }
+
+    if (parentScope) {
+        indentation = parentScope->indentation;
+    }
+
+    // this is used to propagate the scope name right away
+    // so that the newly started operation name shows up
+    SetCompletion(0.0f, "Begin");
+
+    ++indentation;
+}
+
+ProgressScope::~ProgressScope() {
+
+    --indentation;
+
+    if (progressTracker == nullptr)
+        return;
+
+    progressTracker->Lock();
+
+    SetCompletion(1.0f, "End");
+
+    progressTracker->currentScope = parentScope;
+    progressTracker->Unlock();
+
+    progressTracker = nullptr;
+}
+
+void ProgressScope::SetCompletion(float fraction, const char *displayText /*= nullptr*/) {
+
+    if (progressTracker)
+        progressTracker->Lock();
+
+    assert(fraction >= currentCompletion && "Completion progress should always move forwards");
+    assert(fraction >= 0.0f && fraction <= 1.0f && "Completion progress should be between 0 and 1");
+    currentCompletion = fraction;
+
+    if (displayText == nullptr)
+        displayText = "";
+
+    if (parentScope) {
+        parentScope->PropagateChildCompletion(currentCompletion, scopeName, indentation, displayText);
+    } else if (progressTracker) {
+        progressTracker->ProgressUpdate(currentCompletion, scopeName, indentation, displayText);
+    }
+
+    if (progressTracker)
+        progressTracker->Unlock();
+}
+
+void ProgressScope::AddSteps(size_t numSteps) {
+
+    assert(activeStep == -1 && "Steps have to be added at the very beginning");
+
+    const size_t nOld = stepWeights.size();
+    const size_t nNew = nOld + numSteps;
+    stepWeights.resize(nNew);
+
+    for (size_t i = nOld; i < nNew; ++i) {
+        stepWeights[i] = 1.0f;
+    }
+
+    totalExpectedWeight += numSteps;
+}
+
+void ProgressScope::AddStep(float weight /*= 1.0f*/) {
+
+    assert(activeStep == -1 && "Steps have to be added at the very beginning");
+
+    stepWeights.push_back(weight);
+    totalExpectedWeight += weight;
+}
+
+void ProgressScope::StartStep(const char *displayText /*= nullptr*/) {
+
+    if (progressTracker)
+        progressTracker->Lock();
+
+    if (activeStep == -1) {
+        SetCompletion(0.0f, displayText);
+    }
+
+    if (activeStep >= 0 && activeStep < stepWeights.size()) {
+
+        float addCompletion = stepWeights[activeStep] / totalExpectedWeight;
+        SetCompletion(std::min(currentCompletion + addCompletion, 1.0f), displayText);
+
+        baseCompletion = currentCompletion;
+    }
+
+    ++activeStep;
+    assert(activeStep < stepWeights.size() && "Attempting to start more steps than were added");
+
+    if (progressTracker)
+        progressTracker->Unlock();
+}
+
+void ProgressScope::PropagateChildCompletion(float childCompletion, const char *currentScope, int indent, const char *displayText) {
+
+    assert(activeStep < stepWeights.size() && "Not enough steps added to ProgressScope for the number of child scopes used.");
+
+    float curStepWeight = 1.0f;
+
+    if (activeStep >= stepWeights.size())
+        return;
+
+    if (activeStep >= 0) {
+        curStepWeight = stepWeights[activeStep];
+    }
+
+    const float stepCompletion = childCompletion * curStepWeight / totalExpectedWeight;
+    const float totalCompletion = baseCompletion + stepCompletion;
+
+    if (parentScope) {
+        parentScope->PropagateChildCompletion(totalCompletion, currentScope, indent, displayText);
+    } else if (progressTracker) {
+        progressTracker->ProgressUpdate(totalCompletion, currentScope, indent, displayText);
+    }
+}
+
+} // Namespace Assimp

code/PostProcessing/JoinVerticesProcess.cpp

@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "JoinVerticesProcess.h"
 #include "ProcessHelper.h"
 #include <assimp/Vertex.h>
+#include <assimp/ProgressTracker.h>
 #include <assimp/TinyFormatter.h>
 
 #include <stdio.h>
@@ -65,6 +66,8 @@ bool JoinVerticesProcess::IsActive( unsigned int pFlags) const {
 // Executes the post processing step on the given imported data.
 void JoinVerticesProcess::Execute( aiScene* pScene) {
     ASSIMP_LOG_DEBUG("JoinVerticesProcess begin");
+    Assimp::ProgressScope progScope("Join Vertices");
+    progScope.AddSteps(pScene->mNumMeshes);
 
     // get the total number of vertices BEFORE the step is executed
     int iNumOldVertices = 0;
@@ -77,6 +80,7 @@ void JoinVerticesProcess::Execute( aiScene* pScene) {
     // execute the step
     int iNumVertices = 0;
     for (unsigned int a = 0; a < pScene->mNumMeshes; a++) {
+        progScope.StartStep("Joining Mesh Vertices");
         iNumVertices += ProcessMesh(pScene->mMeshes[a], a);
     }
 

include/assimp/ProgressTracker.h

@@ -0,0 +1,188 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2022, assimp team
+
+
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+/** @file ProgressTracker.h
+ */
+
+#pragma once
+#ifndef AI_PROGRESSTRACKER_H
+#define AI_PROGRESSTRACKER_H
+
+#ifdef __GNUC__
+#pragma GCC system_header
+#endif
+
+#include <assimp/types.h>
+
+#include <vector>
+
+namespace Assimp {
+
+// -------------------------------------------------------------------------------
+/**
+ * Abstract base class for receiving progress information.
+ *
+ * Derive and implement the ProgressUpdate() function.
+ * To receive progress updates call ProgressTracker::SetThreadLocalProgressTracker() with your instance.
+ *
+ * On the running thread, this instance will then be used to report back how much of the process has completed and
+ * which operation is currently running.
+ *
+ * Note: Whether any progress gets reported and how detailed the reporting is, depends on how well the executing
+ * code paths are peppered with instances of ProgressScope (see below).
+ *
+ * You can also use ProgressScope in your own code on top of assimp, such that assimp operations become only
+ * one part of the overall progress.
+ */
+// -------------------------------------------------------------------------------
+class ASSIMP_API ProgressTracker {
+
+public:
+    ProgressTracker();
+    virtual ~ProgressTracker();
+
+    /** Makes the given instance the currently active one on this thread.
+     *
+     * This allows to load models on multiple threads and separate their progress reporting.
+     * */
+    static void SetThreadLocalProgressTracker(ProgressTracker *tracker);
+
+    /** In case the derived class needs to access shared resources, this can be used to lock a mutex. */
+    virtual void Lock(){};
+
+    /** In case the derived class needs to access shared resources, this can be used to unlock a mutex. */
+    virtual void Unlock(){};
+
+    /** Called whenever there is a change to the current progress to report
+     *
+     *  @param totalCompletion Value between 0 and 1 that represents how much of all (known) work has been finished.
+     *  @param currentScopeName The name of the ProgressScope that's currently active.
+     *  @param scopeLevel How deep the nesting of ProgressScope's currently is. Can be used for indenting log output.
+     *  @param displayText The text that was passed to ProgressScope::SetCompletion() or ProgressScope::StartStep()
+     *         to show to users, that describes what operation is currently being done.
+     * */
+    virtual void ProgressUpdate(float totalCompletion, const char *currentScopeName, int scopeLevel, const char *displayText) = 0;
+
+private:
+    friend class ProgressScope;
+    ProgressScope *currentScope = nullptr;
+};
+
+// -------------------------------------------------------------------------------
+/**
+ * Instantiate this class locally inside functions to report progress back to the user.
+ *
+ * Suppose you have a function "LoadX" that does three things:
+ * 1) Read a file from disk into memory.
+ * 2) Tokenize the data.
+ * 3) Convert the data to an aiScene.
+ *
+ * To report progress back, instantiate a ProgressScope at the top of the function,
+ * then call AddStep() three times.
+ * If the steps are known to take very different amounts of time, you can give each step a weight.
+ * For example if 1) takes 20% of the time, 2) takes 10% and 3) takes 70%, you can use the step weights 20, 10, 70
+ * or 0.2, 0.1, 0.7. The weights get normalized, so use whatever is more convenient.
+ *
+ * Every time a new 'phase' starts, call StartStep(). This computes the total completion and reports it back
+ * to the user through ProgressTracker.
+ *
+ * Within a step you can use nested ProgressScope's to make progress reporting more fine granular.
+ * For instance within reading the file one could use a nested scope to report back how many bytes have been read.
+ * 
+ * In some cases it is easier to just specify the progress directly, rather than using steps.
+ * Call SetCompletion() in such situations. For example when reading a file, you should not report progress for
+ * every byte read, but you can read an entire chunk of data and then report the percentage.
+ */
+// -------------------------------------------------------------------------------
+class ASSIMP_API ProgressScope final {
+
+public:
+    ProgressScope(const char *scopeName);
+    ~ProgressScope();
+
+    /** Specifies the 0-1 value of progress for this scope directly.
+      *
+      * When using this function, you shouldn't also use steps.
+      *
+      * This function reports the local progress up the chain of parent scopes and combines all their step weights
+      * to ultimately report a single total completion value to ProgressTracker.
+    . */
+    void SetCompletion(float fraction, const char *displayText = nullptr);
+
+    /** Adds a number of steps that are expected to be executed in this scope.
+     *
+     * Each step has equal weighting, meaning it is expected that every step takes approximately the same amount of time.
+     * */
+    void AddSteps(size_t numSteps);
+
+    /** Adds a single step that is expected to be executed in this scope.
+     *
+     * The step can optionally be weighted (all weights are normalized later).
+     * This is used to indicate that a step takes relatively less or more time than other steps within the same scope.
+     * */
+    void AddStep(float weight = 1.0f);
+
+    /** Reports to the ProgressTracker that new progress has been made.
+     *
+     * Set the display string to also indicate what work is currently being done.
+     *
+     * This will compute the overall progress for this scope and call SetCompletion() internally.
+     * */
+    void StartStep(const char *displayText = nullptr);
+
+private:
+    void PropagateChildCompletion(float childCompletion, const char *currentScope, int indent, const char *displayText);
+
+    ProgressScope *parentScope = nullptr;
+    ProgressTracker *progressTracker = nullptr;
+    const char *scopeName = nullptr;
+    int activeStep = -1;
+    std::vector<float> stepWeights;
+    float totalExpectedWeight = 0.0f;
+    float currentCompletion = 0.0f;
+    float baseCompletion = 0.0f;
+    int indentation = 0;
+};
+
+} // Namespace Assimp
+
+#endif // AI_PROGRESSTRACKER_H