fscanner.inl

Go to the documentation of this file.

//==================================================================================================
/// \file
/// This header-file is part of module \alib_files of the \aliblong.
///
/// \emoji :copyright: 2013-2025 A-Worx GmbH, Germany.
/// Published under #"mainpage_license".
//==================================================================================================
ALIB_EXPORT namespace alib { namespace files {
 
#if ALIB_DEBUG
/// The format string used with verbose logging to domain <c>/ALIB/FILES/SCAN</c> by the namespace
/// function #"ScanFiles(FTree&)".<br>
/// Defaults to <c>" {:ta h{2,r} on{10,r} gn{10,r} s(IEC){10,r} dm qqq nf l}"</c>
extern String DBG_FILES_SCAN_VERBOSE_LOG_FORMAT;
#endif
 
 
/// Input parameters to function #"ScanFiles(FTree&)".
struct ScanParameters
{
    /// Options for processing symbolic links.
    enum class SymbolicLinks
    {
        DONT_RESOLVE            = 0, ///< Demands \b not to resolve symbolic links in any way.
        RESOLVE_BUT_DONT_FOLLOW = 1, ///< Demands to read symbolic links, but not follow linked directories.
                                     ///< FInfo dates, sizes, and access rights are set according to
                                     ///< the link target.
        RECURSIVE               = 2, ///< Read symbolic links and in case they are targeting a
                                     ///< directory, recurse into, if this directory meets the
                                     ///< other constraints associated with the current scan.
    };
 
    /// Denotes 'infinite' recursion if set to field #"MaxDepth".
    static constexpr unsigned InfiniteRecursion = (std::numeric_limits<unsigned>::max)();
 
    /// The path to be scanned.
    Path            StartPath;
 
    /// Denotes how symbolic links are treated.
    SymbolicLinks   LinkTreatment                                         =SymbolicLinks::RECURSIVE;
 
    /// The maximum recursion depth. Defaults to #"InfiniteRecursion".
    unsigned        MaxDepth                                                     =InfiniteRecursion;
 
    /// If \c true, the default, scanning does not stop recursion on directories which represent
    /// a mounted filesystem. If \c false, the search is restricted to the device that #"StartPath"
    /// resides in.
    bool            CrossFileSystems                                                         = true;
 
    /// If \c false (the default), scanning aborts if \e 'artificial' filesystems are found.
    /// Artificial filesystems under GNU/Linux, are for example:
    /// <c>/proc</c>, <c>/dev</c>, <c>/run</c>, <c>/sys</c>, and <c>/temp</c>.
    bool            IncludeArtificialFS                                                     = false;
 
    /// If \c false, empty directories remain in the result tree. Otherwise, they are deleted
    /// and do not appear in the tree.
    bool            RemoveEmptyDirectories                                                  = false;
 
    /// If set (not containing \c nullptr), files are passed to this filter and removed if \c false
    /// is returned.<br>
    /// The term "files" here means all sorts of files except Directories.
    /// Directories are either real directories, or in case the field #"LinkTreatment" is set to
    /// #"SymbolicLinks::RECURSIVE", symbolic links that
    /// target a directory.
    ///
    /// \see Optional filters #"DirectoryFilterPreRecursion" and #"DirectoryFilterPostRecursion".
    SPFileFilter    FileFilter;
 
    /// If set (not containing \c nullptr), this filter is invoked \b after a recursive scan of
    /// a directory. If \c false is returned, the recursion is not performed, but the (empty)
    /// directory remains in the result list, if field #"RemoveEmptyDirectories" evaluates to
    /// \c false.<br>
    /// Note that in case field #"LinkTreatment" is set to
    /// #"SymbolicLinks::RECURSIVE", this filter
    /// is also applied to symbolic links, which are readable, not broken, and target a directory.
    ///
    /// \note
    ///   Directories (and symbolic links to directories) are first recursively scanned before this
    ///   filter is applied. On deletion, of course the whole scanned subtree is deleted.
    ///   This allows filtering directories, depending on information available only after
    ///   scanning, hence by the numbers retrieved with #"FInfo::Sums;*".
    ///   To increase performance and filter directories \e before their recursive scan,
    ///   alternative field #"DirectoryFilterPreRecursion" is to be used.
    ///
    /// \see Optional filters #"DirectoryFilterPreRecursion" and #"FileFilter".
    ///
    SPFileFilter    DirectoryFilterPostRecursion;
 
    /// Same as #DirectoryFilterPostRecursion but is used \b before a recursive scan of
    /// a directory. Consequently, this filter leads to much higher scan performance than the
    /// alternative version, because huge branches of the file system might be omitted during scan.
    /// However, the numbers retrieved with #"FInfo::Sums;*" will all indicate
    /// \c 0, because no information is retrieved.<br>
    /// If a directory is "pruned" due to this filter, the entry still occurs in the \b %FTree,
    /// unless field #RemoveEmptyDirectories evaluates to \c true.<br>
    ///
    /// \see Optional filters #DirectoryFilterPostRecursion and #FileFilter.
    ///
    SPFileFilter    DirectoryFilterPreRecursion;
 
    /// Constructor accepting all features.
    /// @param startPath             Stored in field #StartPath.
    /// @param linkTreatment         Stored in field #LinkTreatment. Defaults to \b SymbolicLinks::RECURSIVE.
    /// @param maxDepth              Stored in field #MaxDepth. Defaults to +InfiniteRecursion.
    /// @param crossFileSystems      Stored in field #CrossFileSystems. Defaults to \c true.
    /// @param includeArtificialFS   Stored in field #IncludeArtificialFS. Defaults to \c false.
    ScanParameters( const system::PathString& startPath,
                    SymbolicLinks   linkTreatment      = SymbolicLinks::RECURSIVE,
                    unsigned        maxDepth           = InfiniteRecursion,
                    bool            crossFileSystems   = true,
                    bool            includeArtificialFS= false                        )
    : StartPath          (startPath          )
    , LinkTreatment      (linkTreatment      )
    , MaxDepth           (maxDepth           )
    , CrossFileSystems   (crossFileSystems   )
    , IncludeArtificialFS(includeArtificialFS)                                                    {}
 
}; // struct ScanParameters
 
/// A simple vector containing nodes of an #"FTree". Such nodes are collected during calls
/// of the function #"ScanFiles". One call (aka during the recursive scan of one path) can result
/// in more than one entry in this list, because with resolving symbolic links new isolated
/// siblings can occur.<br>
/// The single new method of this type is #".Add", which checks if the given new start-path
/// is superseding others or is superseded itself by an existing path. In that case the
/// superseded path is deleted.
///
/// Despite the little effort that \alib takes with the provision of these mechanics, often the
/// analysis of, or a loop through this path list is not necessary. This is because most
/// using code would just scan one or more paths and then #"StringTreeIterator;loop through" just
/// all resulting directory and file nodes that have been inserted into the tree.
/// Consequently, the function #"ScanFiles" accepts an instance of this class only optionally.
struct CanonicalPathList : std::vector<FTree::Cursor>
{
    /// Adds the given node to the list, in the case it is not superseded by an already
    /// collected node. Vice versa, existing nodes that are superseded by the given one are removed.
    /// @param node      The node to add.
    ALIB_DLL
    void Add(FTree::Cursor node);
};
 
 
/// ### General Information ###
/// Scans the filesystem according to the given \b ScanParameters and adds #"FInfo"
/// entries to the given #"FTree".
///
/// ### ALib FTree Data Contract ###
/// This function has a contract with the class #"FTree" that is used to store the scan results.
/// This contract states that any file or directory found during a scan is always stored using
/// the <em>"Real Path"</em> of the entry. This means that any symbolic link is resolved.
/// The consequences are:
/// - %Files and directories which represent a symbolic link are always "leaf nodes".
///   (They never contain child nodes.). However, their symlink target path is attached twice
///   to the entry:
///   1. The original link information given, which often uses relative path addressing.
///   2. The absolute, <em>"Real Path"</em> of the target, which has a corresponding result entry
///      in the given \b %FTree.
/// - If a using software wants to use symbolic paths, for example, to present them to the end
///   user, such paths have to be assembled by the user's code in own responsibility.
///   All information for doing this is provided in the resulting tree object
/// - Doubly linked target files and directories are never a problem for this scanner. Each
///   file is scanned only once. This especially prevents all sorts of problems that would otherwise
///   occur with cyclic symbolic links.
/// - Due to this, even the given start path of a search might not be found as a result
///   in the given \b %FTree, because also start paths are converted to a <em>Real Path</em>.
/// - The scan result may contain more than one resulting path. This happens if a symbolic link
///   targets a file or directory not recursively included in the start path.
///   The resulting <em>"Real Path"</em> of the given start path is, however, always the first
///   result added.
///
/// The latter is reflected with (optional) parameter \p{resultPaths} of this function, which is
/// of type #"CanonicalPathList".
///
/// \note
///   Because the class #"FTree" is based on class #"StringTree", using code
///   is enabled to break this contract by adding entries below symbolic links.
///   Other entities of this \alibmod_nl will not break this contract.
///
/// ### Rescanning of Entries ###
/// Existing entries in the given \p{tree} are not overwritten. They might be scanned with "higher"
/// #"FInfo::ScanStates;*" values, depending on given \p{parameters} and how they had been
/// scanned before. If the same "level" of scanning is provided,  existing entries will not be
/// scanned again. If a rescan of a certain path is wanted, then the target entry of that path has
/// to be deleted before invoking this function. Due to the implementation of class FTree, repeated
/// delete and scan operations will not cause any heap-memory allocations or deallocations.
///
/// ### platform-dependent Code Selection ###
/// File scanning is a platform-dependent task and hence \b ALib uses one of two different
/// implementations:
/// 1. A posix version for posix compatible OSes,
/// 2. A version that relies on <c>C++ std::filesystem</c>.
///
/// The fallback version using <c>std::filesystem</c> has the following restrictions:
/// - The only time attribute available is the #"FInfo::MDate;modification time" of
///   an entry. The fields #"FInfo::BDate", #"FInfo::CDate", and #"FInfo::ADate" are always set
///   to the same as the modification time, even on filesystems that support the other values.
/// - The file time of symbolic links is \b always that of the target file. The C++ standard has
///   no possibility to access the link's time itself.
/// - The file time of broken symbolic links is set to the current time (time of scanning).
/// - The size that directories occupy on a disk cannot be determined.
///   Directory entries always report size <c>0</c>.
/// - The target of a symbolic link which points to a non-accessible directory, cannot be resolved
///   to a "real" (aka canonical) path, even if all other path components before were accessible.
///   (This is true for the implementation of the standard library under GNU/Linux and Clang
///   compiler at the time of writing this, 2024/02.)
/// - The flag #"ScanParameters::CrossFileSystems;*" is ignored. Crossing Filesystems cannot
///   be detected using purely the standard library.
/// - A files' owner and owning group is not determined. Instead, #"FInfo::UnknownID;*" is set for
///   both.
/// - The scanning process is half as fast as in the Posix version. The reason for this is probably
///   the internal allocation and deallocation of many quite volatile string objects in the C++
///   standard library.
///   Well, but it is still fast though!
///
/// \note As for today, using this module under WindowsOS, will fall back to the
///       <em>C++ std::filesystem</em> version. It may be that a future version will provide a
///       native implementation of this target system. Volunteers from the community are welcome to
///       contribute.
///
/// @param      tree           The tree to fill.
/// @param      parameters     The input parameters to determine the scan process.
/// @param[out] resultPaths    An optional container to store the result paths of a scan.
///                            If \c nullptr is given, the result paths are not collected. See the
///                            #"CanonicalPathList;types documentation" for further information.
/// @param[out] remainingStart An optional path string. If given, on failure, it will receive the
///                            remainder of the path given with #"ScanParameters::StartPath;2"
///                            starting with the first directory or file that could not be resolved
///                            or accessed.
///
/// @return The scan state code of the tree node of the first resulting path, hence of the node
///         referred to by the given #"ScanParameters::StartPath;2".<br>
///         On error, i.e. if the start path was invalid, not accessible, a broken link, a circular
///         link, or other failures, #"ScanStates::NOT_EXISTENT" is returned.
ALIB_DLL
FInfo::ScanStates  ScanFiles( FTree&                      tree,
                             ScanParameters&             parameters,
                             CanonicalPathList*          resultPaths       = nullptr,
                             Path*                       remainingStart = nullptr   );
 
/// Analyses the given \p{sourcePath} and converts it to its canonical version.
/// This is similar to what the posix function <c>realpath()</c> and C++
/// <c>std::filesystem::canonical</c> do.<br>
/// This version, in addition, creates corresponding nodes in the #"FTree" (passed indirectly with
/// the parameter \p{node}). Besides removing <c>"."</c> and <c>".."</c> entries, symbolic links
/// are not only resolved, but the nodes they are targeting receive information about the link
/// that targeted them. This information is set with the method #"File::SetSymbolicParent(File)".
/// With that, the path of directories or files that are children of such targeted node, can
/// re-establish the file-path as originally specified. This is done with the method
/// #"File::AssembleSymbolicPath".
///
/// \note
///   This function is mainly used by the function #"ScanFiles" and a direct use is seldom needed.
/// @param[in,out] sourcePath  The path to scan. This might contain <c>"."</c> and <c>".."</c>
///                            directories, as well as symbolic links.
///                            When the method exits successfully, this path is empty when the
///                            method returns. Otherwise, this path-string contains the remaining
///                            path, starting with the name of the file or directory, that could
///                            not be found, accessed, or otherwise be resolved.
/// @param[in,out] node        The starting node. In case the parameter \p{sourcePath} is an
///                            absolute path, this node is changed to the
///                            #"TCursor::GoToRoot;root folder" of the #"FTree".<br>
///                            When the method exits successfully, this cursor targets the
///                            file that the source path resolved to.<br>
///                            In case of failure, this cursor becomes
///                            #"TCursor::IsInvalid;invalid".
/// @param[in,out] pathToNode  This path has to point to the given \p{node} when the method is
///                            called. When the method returns, it points to the then moved
///                            \p{node}.
///                            When the method exits this contains the path to the modified \p{node}
///                            not scanned, yet.
/// @param[in,out] resultPaths Todo
/// @return The scan state of the target node. This should usually be #"FInfo::ScanStates::STATS"
///         or #"FInfo::ScanStates::RESOLVED". In case of failure,
///         #"FInfo::ScanStates::NOT_EXISTENT" or other scan state values that indicate failure
///         are returned.
FInfo::ScanStates  MakeCanonical(  Path&               sourcePath,
                                  FTree::Cursor&      node,
                                  Path&               pathToNode,
                                  CanonicalPathList*  resultPaths= nullptr   );
 
} // namespace alib[::files]
 
 
/// Type alias in namespace \b alib.
using     ScanParameters    =   files::ScanParameters;
 
/// Type alias in namespace \b alib.
using     CanonicalPathList =   files::CanonicalPathList;
 
}  // namespace [alib]