diff --git a/Project.toml b/Project.toml index e5959e06..6397e746 100644 --- a/Project.toml +++ b/Project.toml @@ -23,6 +23,7 @@ RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +StructUtils = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b" @@ -64,6 +65,7 @@ SHA = "0.7, 1" SparseArrays = "1.11" StaticArrays = "1" Statistics = "1.11" +StructUtils = "2.6.0" Tables = "1.11.1" Tar = "1.9" TensorCast = "0.3.3, 0.4" diff --git a/docs/make.jl b/docs/make.jl index 14ec53c6..8c0a8238 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -19,8 +19,8 @@ makedocs(; ], "Reference" => ["func_ref.md", "services_ref.md", "blob_ref.md"], ], + checkdocs = :public, # warnonly=[:doctest], - # checkdocs=:none, # html_prettyurls = !("local" in ARGS), ) diff --git a/docs/src/GraphData.md b/docs/src/GraphData.md index 3b710564..97072ca3 100644 --- a/docs/src/GraphData.md +++ b/docs/src/GraphData.md @@ -44,7 +44,19 @@ Labels are the principle identifier of a variable or factor. #### Timestamps -Each variable or factor can have a timestamp associated with it. +Each variable or factor can have a timestamp associated with it representing when the physical event/observation occurred. + +**Time Standard**: Timestamps use UTC-based time (Coordinated Universal Time) via the `TimeDateZone` type, not TAI (International Atomic Time). The key difference is that UTC includes leap seconds to keep synchronized with Earth's rotation, while TAI is a continuous monotonic time scale without leap seconds. + +**Timezone Support**: While timestamps default to UTC (`tz"UTC"`), `TimeDateZone` supports any timezone (e.g., `tz"America/New_York"`, `tz"Europe/London"`). The timezone offset is preserved when stored and retrieved. + +**Event Time vs Database Time**: The timestamp represents the physical event time (when a sensor measurement was taken, when a robot was at a pose, etc.), not when the variable was added to the database. Database metadata timestamps may be tracked separately by specific backend implementations. + +**Temporal Uncertainty**: This single timestamp value is metadata and does not represent temporal uncertainty. For problems where time itself is a state variable requiring inference (e.g., using `SGal3` which includes temporal components), include time as part of your state type rather than relying on this metadata field. Furthermore, the concept of "ClockPriors" is also at play for the API design changes relating to v2. + +**Future**: changing away from field name `timestamp` was deferred from v1.0 owing to resolution of older complexity of other fields being standardized; however, the desire for a better name exists and should be revisited in the v2 API. At time of writing, the best summary of the naming debate is captured here: https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/1087#issuecomment-3582252305. In the lead up to a new long term release (a.k.a semver major), a deprecation cycle will be used via semver minors to ensure users have an easy and macro-aided transition in the API. + +Related functions: - [`getTimestamp`](@ref) diff --git a/src/Common.jl b/src/Common.jl index 8fe4b071..86d6337c 100644 --- a/src/Common.jl +++ b/src/Common.jl @@ -187,7 +187,20 @@ function calcDeltatime(from::TimeDateZone, to::TimeDateZone) end calcDeltatime(from_node, to_node) = calcDeltatime(from_node.timestamp, to_node.timestamp) -function tdz_now(zone = tz"UTC") #TODO or default to slower localzone()? +Timestamp(args...) = TimeDateZone(args...) +Timestamp(t::Nanosecond, zone = tz"UTC") = Timestamp(Val(:unix), t, zone) +function Timestamp(epoch::Val{:unix}, t::Nanosecond, zone = tz"UTC") + return TimeDateZone(TimeDate(1970) + t, zone) +end +function Timestamp(epoch::Val{:unix}, t::Float64, zone = tz"UTC") + return Timestamp(epoch, Nanosecond(t * 10^9), zone) +end +Timestamp(t::Float64, zone = tz"UTC") = Timestamp(Val(:unix), t, zone) +function Timestamp(epoch::Val{:rata}, t::Float64, zone = tz"UTC") + return TimeDateZone(convert(DateTime, Millisecond(t*10^3)), zone) +end + +function now_tdz(zone = tz"UTC") t = time() - return TimeDateZone(TimeDate(1970) + Nanosecond(t * 10^9), zone) + return Timestamp(t, zone) end diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index 14a6e8b1..9aaf2dd7 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -44,7 +44,7 @@ StructUtils.@kwarg struct Blobentry """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple.""" metadata::JSONText = JSONText("{}") """ When the Blob itself was first created. Serialized as an ISO 8601 string.""" - timestamp::TimeDateZone = tdz_now() + timestamp::TimeDateZone = now_tdz() """ Type version of this Blobentry.""" version::VersionNumber = DFG.version(Blobentry) end diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 54f92e4d..7219f82f 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -317,10 +317,6 @@ function getFactorState(args...) ) end -function updateBlob!(args...) - return error("updateBlob! is obsolete as blobid=>Blob pairs are immutable.") -end - function setTags!(node, tags::Union{Vector{Symbol}, Set{Symbol}}) Base.depwarn("setTags! is deprecated, use mergeTags! or addTags! instead.", :setTags!) node.tags !== tags && empty!(node.tags) @@ -343,7 +339,6 @@ const DFGFactorSummary = FactorSummary const DFGFactor = FactorCompute const PackedFactor = FactorDFG const Factor = FactorDFG -const SmallDataTypes = MetadataTypes const AbstractPrior = PriorObservation const AbstractRelative = RelativeObservation const AbstractParams = AbstractDFGParams @@ -462,14 +457,6 @@ function getDFGInfo(dfg::AbstractDFG) ) end -# """ -# $TYPEDSIGNATURES -# List all the solvekeys used amongst all variables in the distributed factor graph object. - -# Related - -# [`listSolveKeys`](@ref), [`refStates`](@ref), [`listVariables`](@ref) -# """ function listSolveKeys( variable::VariableCompute, filterSolveKeys::Union{Regex, Nothing} = nothing, @@ -522,15 +509,7 @@ const listSupersolves = listSolveKeys #TODO mergeBlobentries! does not fit with merge definition, should probably be updated to copyto or sync. # leaving here until it is done. - -# """ -# $SIGNATURES - -# Add a blob entry into the destination variable which already exists -# in a source variable. - -# See also: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`listBlobentries`](@ref), [`getBlob`](@ref) -# """ +# Add a blob entry into the destination variable which already exists in a source variable. function mergeBlobentries!( dst::AbstractDFG, dlbl::Symbol, @@ -583,15 +562,6 @@ function mergeBlobentries!( return varList end -# """ -# $(SIGNATURES) - -# Get all blob entries matching a Regex pattern over variables - -# Notes -# - Use `dropEmpties=true` to not include empty lists in result. -# - Use keyword `varList` for which variables to search through. -# """ function getBlobentriesVariables( dfg::AbstractDFG, bLblPattern::Regex; @@ -621,13 +591,9 @@ function getBlobentries(dfg::AbstractDFG, label::Symbol, regex::Regex) return entries = getBlobentries(dfg, label; labelFilter = contains(regex)) end -function getBlobentries( - dfg::AbstractDFG, - label::Symbol, - skey::Union{Symbol, <:AbstractString}, -) +function getBlobentries(dfg::AbstractDFG, label::Symbol, skey::AbstractString) Base.depwarn( - "getBlobentries(dfg, label, ::Union{Symbol, <:AbstractString}) is deprecated, use getBlobentries(dfg, label; labelFilter=contains(regex)) instead.", + "getBlobentries(dfg, label, regex::AbstractString) is deprecated, use getBlobentries(dfg, label; labelFilter=contains(regex)) instead.", :getBlobentries, ) return getBlobentries(dfg, label, Regex(string(skey))) @@ -674,666 +640,8 @@ end setMetadata!(args...) = error("setMetadata is obsolete, use Bloblets instead.") -function updateData!( - dfg::AbstractDFG, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = true, -) - @warn "updateData! is obsolete." - checkhash && assertHash(entry, blob; hashfunction) - # order of ops with unknown new blobId not tested - mergeBlobentry!(dfg, label, entry) - db = updateBlob!(dfg, de, blob) - return 2 -end - -function updateData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, -) - @warn "updateData! is obsolete." - # Recalculate the hash - NOTE Assuming that this is going to be a Blobentry. TBD. - # order of operations with unknown new blobId not tested - newEntry = Blobentry( - entry; # and kwargs to override new values - blobstore = getLabel(blobstore), - hash = string(bytes2hex(hashfunction(blob))), - origin = buildSourceString(dfg, label), - _version = _getDFGVersion(), - ) - mergeBlobentry!(dfg, label, newEntry) - updateBlob!(blobstore, newEntry, blob) - return 2 -end - -function updateBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T} - @warn "updateBlob! is obsolete." - if haskey(store.blobs, blobId) - @warn "Key '$blobId' doesn't exist." - end - return store.blobs[blobId] = RowBlob(blobId, blob) -end - -function getData( - dfg::AbstractDFG, - vlabel::Symbol, - key::Union{Symbol, UUID, <:AbstractString, Regex}; - hashfunction = sha256, - checkhash::Bool = true, - getlast::Bool = true, -) - Base.depwarn("getData is deprecated, use loadBlob_Variable instead.", :getData) - _getblobentr(g, v, k) = getBlobentries(g, v, k) - _getblobentr(g, v, k::UUID) = [getfirstBlobentry(g, v, k);] - de_ = _getblobentr(dfg, vlabel, key) - lbls = (s -> s.label).(de_) - idx = sortperm(lbls; rev = getlast) - _first(s) = s - _first(s::AbstractVector) = 0 < length(s) ? s[1] : nothing - de = _first(de_[idx]) - if isnothing(de) - @error "Could not find in $vlabel the key $key" - return nothing - end - db = getBlob(dfg, de) - - checkhash && assertHash(de, db; hashfunction = hashfunction) - return de => db -end - -# This is the normal one -function getData( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - var_label::Symbol, - entry_label::Symbol; - hashfunction = sha256, - checkhash::Bool = true, - getlast::Bool = true, -) - Base.depwarn("getData is deprecated, use loadBlob_Variable instead.", :getData) - de = getBlobentry(dfg, var_label, entry_label) - db = getBlob(blobstore, de) - checkhash && assertHash(de, db; hashfunction) - return de => db -end - -#FIXME Should `addData!`` not return entry=>blob pair? -function addData!( - dfg::AbstractDFG, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = false, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(dfg, entry, blob) |> UUID - newEntry = Blobentry(entry; blobId) #, size=length(blob)) - return addBlobentry!(dfg, label, newEntry) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = false, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(blobstore, entry, blob) |> UUID - newEntry = Blobentry(entry; blobId) #, size=length(blob)) - return addBlobentry!(dfg, label, newEntry) -end - -function addData!( - dfg::AbstractDFG, - blobstorekey::Symbol, - vLbl::Symbol, - bLbl::Symbol, - blob::Vector{UInt8}, - timestamp = now(localzone()); - kwargs..., -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - return addData!( - dfg, - getBlobstore(dfg, blobstorekey), - vLbl, - bLbl, - blob, - timestamp; - kwargs..., - ) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - bLbl::Symbol, - blob::Vector{UInt8}, - timestamp = now(localzone()); - description = "", - metadata = "", - mimeType::String = "application/octet-stream", - id::Union{UUID, Nothing} = nothing, - blobId::UUID = uuid4(), - hashfunction = sha256, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - # - entry = Blobentry(; - id, - blobId, - label = bLbl, - blobstore = getLabel(blobstore), - hash = string(bytes2hex(hashfunction(blob))), - origin = buildSourceString(dfg, vLbl), - description, - mimeType, - metadata, - timestamp, - ) - - return addData!(dfg, blobstore, vLbl, entry, blob; hashfunction) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore{T}, - vLbl::Symbol, - blobLabel::Symbol, - blob::T, - timestamp = now(localzone()); - description = "", - metadata = "", - mimeType::String = "application/octet-stream", - origin = buildSourceString(dfg, vLbl), - # hashfunction = sha256, -) where {T} - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - # - # checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(blobstore, blob) - - entry = Blobentry(; - blobId, - label = blobLabel, - blobstore = getLabel(blobstore), - # hash = string(bytes2hex(hashfunction(blob))), - hash = "", - origin, - description, - mimeType, - metadata, - timestamp, - ) - addBlobentry!(dfg, vLbl, entry) - return entry => blob -end - -function deleteData!(dfg::AbstractDFG, vLbl::Symbol, bLbl::Symbol) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - de = getBlobentry(dfg, vLbl, bLbl) - deleteBlobentry!(dfg, vLbl, bLbl) - deleteBlob!(dfg, de) - return 2 -end - -function deleteData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - entry::Blobentry, -) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - return deleteData!(dfg, blobstore, vLbl, entry.label) -end - -function deleteData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - bLbl::Symbol, -) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - de = getBlobentry(dfg, vLbl, bLbl) - deleteBlobentry!(dfg, vLbl, bLbl) - deleteBlob!(blobstore, de) - return 2 -end - -## ================================================================================ -## Deprecated in v0.27 -##================================================================================= - -# const AbstractFactor = AbstractObservation -# const AbstractPackedFactor = AbstractPackedObservation -# const FactorOperationalMemory = FactorCache -# const VariableNodeData = State - -# @deprecate getNeighborhood(args...; kwargs...) listNeighborhood(args...; kwargs...) -# @deprecate addBlob!(store::AbstractBlobstore, blobId::UUID, data, ::String) addBlob!( -# store, -# blobId, -# data, -# ) -# @deprecate addBlob!(store::AbstractBlobstore{T}, data::T, ::String) where {T} addBlob!( -# store, -# uuid4(), -# data, -# ) - -# @deprecate updateVariable!(args...) mergeVariable!(args...) -# @deprecate updateFactor!(args...) mergeFactor!(args...) - -# @deprecate updateBlobEntry!(args...) mergeBlobentry!(args...) -# @deprecate updateGraphBlobEntry!(args...) mergeGraphBlobentry!(args...) -# @deprecate updateAgentBlobEntry!(args...) mergeAgentBlobentry!(args...) - -# @deprecate getBlobStore(args...) getBlobstore(args...) -# @deprecate addBlobStore!(args...) addBlobstore!(args...) -# @deprecate updateBlobStore!(args...) updateBlobstore!(args...) -# @deprecate deleteBlobStore!(args...) deleteBlobstore!(args...) -# @deprecate emptyBlobStore!(args...) emptyBlobstore!(args...) -# @deprecate listBlobStores(args...) listBlobstores(args...) - -# @deprecate BlobEntry(args...; kwargs...) Blobentry(args...; kwargs...) -# @deprecate getGraphBlobEntry(args...; kwargs...) getGraphBlobentry(args...; kwargs...) -# @deprecate getGraphBlobEntries(args...; kwargs...) getGraphBlobentries(args...; kwargs...) -# @deprecate addGraphBlobEntry!(args...; kwargs...) addGraphBlobentry!(args...; kwargs...) -# @deprecate addGraphBlobEntries!(args...; kwargs...) addGraphBlobentries!(args...; kwargs...) -# @deprecate mergeGraphBlobEntry!(args...; kwargs...) mergeGraphBlobentry!(args...; kwargs...) -# @deprecate deleteGraphBlobEntry!(args...; kwargs...) deleteGraphBlobentry!( -# args...; -# kwargs..., -# ) -# @deprecate getAgentBlobEntry(args...; kwargs...) getAgentBlobentry(args...; kwargs...) -# @deprecate getAgentBlobEntries(args...; kwargs...) getAgentBlobentries(args...; kwargs...) -# @deprecate addAgentBlobEntry!(args...; kwargs...) addAgentBlobentry!(args...; kwargs...) -# @deprecate addAgentBlobEntries!(args...; kwargs...) addAgentBlobentries!(args...; kwargs...) -# @deprecate mergeAgentBlobEntry!(args...; kwargs...) mergeAgentBlobentry!(args...; kwargs...) -# @deprecate deleteAgentBlobEntry!(args...; kwargs...) deleteAgentBlobentry!( -# args...; -# kwargs..., -# ) -# @deprecate listGraphBlobEntries(args...; kwargs...) listGraphBlobentries(args...; kwargs...) -# @deprecate listAgentBlobEntries(args...; kwargs...) listAgentBlobentries(args...; kwargs...) -# @deprecate hasBlobEntry(args...; kwargs...) hasBlobentry(args...; kwargs...) -# @deprecate getBlobEntry(args...; kwargs...) getBlobentry(args...; kwargs...) -# @deprecate getBlobEntryFirst(args...; kwargs...) getfirstBlobentry(args...; kwargs...) -# @deprecate getBlobentry(var::AbstractGraphVariable, blobId::UUID) getfirstBlobentry( -# var::AbstractGraphVariable, -# blobId::UUID, -# ) -# @deprecate addBlobEntry!(args...; kwargs...) addBlobentry!(args...; kwargs...) -# @deprecate addBlobEntries!(args...; kwargs...) addBlobentries!(args...; kwargs...) -# @deprecate mergeBlobEntry!(args...; kwargs...) mergeBlobentry!(args...; kwargs...) -# @deprecate deleteBlobEntry!(args...; kwargs...) deleteBlobentry!(args...; kwargs...) -# @deprecate listBlobEntrySequence(args...; kwargs...) listBlobentrySequence( -# args...; -# kwargs..., -# ) -# @deprecate mergeBlobEntries!(args...; kwargs...) mergeBlobentries!(args...; kwargs...) - -# @deprecate getVariableSolverData(args...; kwargs...) getState(args...; kwargs...) -# @deprecate addVariableSolverData!(args...; kwargs...) addState!(args...; kwargs...) -# @deprecate deleteVariableSolverData!(args...; kwargs...) deleteState!(args...; kwargs...) -# @deprecate listVariableSolverData(args...; kwargs...) listStates(args...; kwargs...) -# @deprecate getVariableSolverDataAll(args...; kwargs...) getStates(args...; kwargs...) - -# @deprecate getSolverData(v::VariableCompute, solveKey::Symbol = :default) getState( -# v, -# solveKey, -# ) false - -# @deprecate packVariableNodeData(args...; kwargs...) packState(args...; kwargs...) -# @deprecate unpackVariableNodeData(args...; kwargs...) unpackState(args...; kwargs...) - -# #TODO possibly completely deprecated or not exported until update verb is standardized -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# variablekey::Symbol, -# vnd::State, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# Base.depwarn( -# "updateVariableSolverData! is deprecated, use mergeState! or copytoState! instead", -# :updateVariableSolverData!, -# ) -# #This is basically just setSolverData -# var = getVariable(dfg, variablekey) -# warn_if_absent && -# !haskey(var.solverDataDict, vnd.solveKey) && -# @warn "State '$(vnd.solveKey)' does not exist, adding" - -# # for InMemoryDFGTypes do memory copy or repointing, for cloud this would be an different kind of update. -# usevnd = vnd # useCopy ? deepcopy(vnd) : vnd -# # should just one, or many pointers be updated? -# useExisting = -# haskey(var.solverDataDict, vnd.solveKey) && -# isa(var.solverDataDict[vnd.solveKey], State) && -# length(fields) != 0 -# # @error useExisting vnd.solveKey -# if useExisting -# # change multiple pointers inside the VND var.solverDataDict[solvekey] -# for field in fields -# destField = getfield(var.solverDataDict[vnd.solveKey], field) -# srcField = getfield(usevnd, field) -# if isa(destField, Array) && size(destField) == size(srcField) -# # use broadcast (in-place operation) -# destField .= srcField -# else -# # change pointer of destination VND object member -# setfield!(var.solverDataDict[vnd.solveKey], field, srcField) -# end -# end -# else -# # change a single pointer in var.solverDataDict -# var.solverDataDict[vnd.solveKey] = usevnd -# end - -# return var.solverDataDict[vnd.solveKey] -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# variablekey::Symbol, -# vnd::State, -# solveKey::Symbol, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# # TODO not very clean -# if vnd.solveKey != solveKey -# Base.depwarn( -# "updateVariableSolverData with solveKey is deprecated use copytoState! instead.", -# :updateVariableSolverData!, -# ) -# usevnd = useCopy ? deepcopy(vnd) : vnd -# usevnd.solveKey = solveKey -# return updateVariableSolverData!( -# dfg, -# variablekey, -# usevnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# else -# return updateVariableSolverData!( -# dfg, -# variablekey, -# vnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# sourceVariable::VariableCompute, -# solveKey::Symbol = :default, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# # -# vnd = getSolverData(sourceVariable, solveKey) -# # toshow = listSolveKeys(sourceVariable) |> collect -# # @info "update DFGVar solveKey" solveKey vnd.solveKey -# # @show toshow -# @assert solveKey == vnd.solveKey "State's solveKey=:$(vnd.solveKey) does not match requested :$solveKey" -# return updateVariableSolverData!( -# dfg, -# sourceVariable.label, -# vnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# sourceVariables::Vector{<:VariableCompute}, -# solveKey::Symbol = :default, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# #I think cloud would do this in bulk for speed -# for var in sourceVariables -# updateVariableSolverData!( -# dfg, -# var.label, -# getSolverData(var, solveKey), -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end -# end - -# ## factor refactor deprecations -# Base.@kwdef mutable struct GenericFunctionNodeData{ -# T <: Union{<:AbstractPackedObservation, <:AbstractObservation, <:FactorCache}, -# } -# eliminated::Bool = false -# potentialused::Bool = false -# edgeIDs::Vector{Int} = Int[] -# fnc::T -# multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 -# certainhypo::Vector{Int} = Int[] -# nullhypo::Float64 = 0.0 -# solveInProgress::Int = 0 -# inflation::Float64 = 0.0 -# end - -# function FactorCompute( -# label::Symbol, -# timestamp::Union{DateTime, ZonedDateTime}, -# nstime::Nanosecond, -# tags::Set{Symbol}, -# solverData::GenericFunctionNodeData, -# solvable::Int, -# variableOrder::Union{Vector{Symbol}, Tuple}; -# observation = getFactorType(solverData), -# state::FactorState = FactorState(), -# solvercache::Base.RefValue{<:FactorCache} = Ref{FactorCache}(), -# id::Union{UUID, Nothing} = nothing, -# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -# ) -# error( -# "This constructor is deprecated, use FactorCompute(label, variableOrder, solverData; ...) instead", -# ) -# return FactorCompute( -# id, -# label, -# tags, -# Tuple(variableOrder), -# timestamp, -# nstime, -# Ref(solverData), -# Ref(solvable), -# smallData, -# observation, -# state, -# solvercache, -# ) -# end - -# function getSolverData(f::FactorCompute) -# return error( -# "getSolverData(f::FactorCompute) is obsolete, use getFactorState, getObservation, or getCache instead", -# ) -# end - -# function setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) -# return error( -# "setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) is obsolete, use setState!, or setCache! instead", -# ) -# end - -# @deprecate unpackFactor(dfg::AbstractDFG, factor::FactorDFG; skipVersionCheck::Bool = false) unpackFactor( -# factor; -# skipVersionCheck, -# ) false - -# @deprecate rebuildFactorMetadata!(args...; kwargs...) rebuildFactorCache!( -# args...; -# kwargs..., -# ) - -# function reconstFactorData end - -# function decodePackedType( -# dfg::AbstractDFG, -# varOrder::AbstractVector{Symbol}, -# ::Type{T}, -# packeddata::GenericFunctionNodeData{PT}, -# ) where {T <: FactorCache, PT} -# error("decodePackedType is obsolete") -# # -# # TODO, to solve IIF 1424 -# # variables = map(lb->getVariable(dfg, lb), varOrder) - -# # Also look at parentmodule -# usrtyp = convertStructType(PT) -# fulltype = DFG.FunctionNodeData{T{usrtyp}} -# factordata = reconstFactorData(dfg, varOrder, fulltype, packeddata) -# return factordata -# end - -# function _packSolverData(f::FactorCompute, fnctype::AbstractObservation) -# # -# error("_packSolverData is deprecated, use seperate packing of observation #TODO") -# packtype = convertPackedType(fnctype) -# try -# packed = convert(PackedFunctionNodeData{packtype}, getSolverData(f)) #TODO getSolverData -# packedJson = packed -# return packedJson -# catch ex -# io = IOBuffer() -# showerror(io, ex, catch_backtrace()) -# err = String(take!(io)) -# msg = "Error while packing '$(f.label)' as '$fnctype', please check the unpacking/packing converters for this factor - \r\n$err" -# error(msg) -# end -# end - -# const PackedFunctionNodeData{T} = -# GenericFunctionNodeData{T} where {T <: AbstractPackedObservation} -# function PackedFunctionNodeData(args...; kw...) -# error("PackedFunctionNodeData is obsolete") -# return PackedFunctionNodeData{typeof(args[4])}(args...; kw...) -# end - -# const FunctionNodeData{T} = -# GenericFunctionNodeData{T} where {T <: Union{<:AbstractObservation, <:FactorCache}} -# FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw...) - -# # this is the GenericFunctionNodeData for packed types -# #TODO deprecate FactorData in favor of FactorState (with no more distinction between packed and compute) -# const FactorData = PackedFunctionNodeData{AbstractPackedObservation} - -# function FactorCompute( -# label::Symbol, -# variableOrder::Union{Vector{Symbol}, Tuple}, -# solverData::GenericFunctionNodeData; -# tags::Set{Symbol} = Set{Symbol}(), -# timestamp::Union{DateTime, ZonedDateTime} = now(localzone()), -# solvable::Int = 1, -# nstime::Nanosecond = Nanosecond(0), -# id::Union{UUID, Nothing} = nothing, -# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -# ) -# Base.depwarn( -# "`FactorCompute` constructor with `GenericFunctionNodeData` is deprecated. observation, state, and solvercache should be provided explicitly.", -# :FactorCompute, -# ) -# observation = getFactorType(solverData) -# state = FactorState( -# solverData.eliminated, -# solverData.potentialused, -# solverData.multihypo, -# solverData.certainhypo, -# solverData.nullhypo, -# solverData.solveInProgress, -# solverData.inflation, -# ) - -# if solverData.fnc isa FactorCache -# solvercache = solverData.fnc -# else -# solvercache = nothing -# end - -# return FactorCompute( -# label, -# Tuple(variableOrder), -# observation, -# state, -# solvercache; -# id, -# timestamp, -# nstime, -# tags, -# smallData, -# solvable, -# ) -# end - -# # Deprecated check usefull? # packedFnc = fncStringToData(factor.fnctype, factor.data) -# # Deprecated check usefull? # decodeType = getFactorOperationalMemoryType(dfg) -# # Deprecated check usefull? # fullFactorData = decodePackedType(dfg, factor.variableorder, decodeType, packedFnc) -# function fncStringToData(args...; kwargs...) -# @warn "fncStringToData is obsolete, called with" args kwargs -# return error("fncStringToData is obsolete.") -# end - -# #TODO make sure getFactorOperationalMemoryType is obsolete -# function getFactorOperationalMemoryType(dummy) -# return error( -# "Please extend your workspace with function getFactorOperationalMemoryType(<:AbstractParams) for your usecase, e.g. IncrementalInference uses `CommonConvWrapper <: FactorCache`", -# ) -# end -# function getFactorOperationalMemoryType(dfg::AbstractDFG) -# return getFactorOperationalMemoryType(getSolverParams(dfg)) -# end - -# function typeModuleName(variableType::StateType) -# Base.depwarn("typeModuleName is obsolete", :typeModuleName) -# io = IOBuffer() -# ioc = IOContext(io, :module => DistributedFactorGraphs) -# show(ioc, typeof(variableType)) -# return String(take!(io)) -# end - -# typeModuleName(varT::Type{<:StateType}) = typeModuleName(varT()) +updateData!(args...; kwargs...) = error("updateData! is obsolete.") +updateBlob!(args...; kwargs...) = error("updateBlob! is obsolete.") +getData(args...; kwargs...) = error("getData is obsolete, use loadBlob_Variable") +addData!(args...; kwargs...) = error("addData! is obsolete, use saveBlob_Variable!") +deleteData!(args...; kwargs...) = error("deleteData! is obsolete, use deleteBlob_Variable!") diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 8afc9ef5..9931a966 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -52,6 +52,10 @@ using StaticArrays using InteractiveUtils: subtypes +using StructUtils: @kwarg, @tags +public @tags +public @kwarg + ##============================================================================== # Exports ##============================================================================== @@ -582,7 +586,7 @@ const unstable_functions::Vector{Symbol} = [ # :listPPEs, # :mergePPEs!, Symbol("@defVariable"), - :SmallDataTypes, + # :SmallDataTypes, :NoSolverParams, :AbstractParams, # Deprecated in v0.27 diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 3c01cb38..ce523429 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -43,7 +43,7 @@ end # *not available without reconstruction # Packed Factor constructor -function assembleFactorName(xisyms::Union{Vector{String}, Vector{Symbol}}) +function assembleFactorName(xisyms) return Symbol(xisyms..., "_f", randstring(4)) end @@ -68,7 +68,7 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap variableorder::NTuple{N, Symbol} & (choosetype = x->NTuple{length(x), Symbol},) # NOTE v0.29 renamed from _variableOrderSymbols """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::TimeDateZone = tdz_now() # NOTE v0.29 changed from ZonedDateTime + timestamp::TimeDateZone = now_tdz() # NOTE v0.29 changed from ZonedDateTime # TODO # """(Optional) Steady (monotonic) time in nanoseconds `Nanosecond` (`Int64``)""" # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime. @@ -103,7 +103,7 @@ function FactorDFG( variableorder::Union{<:Tuple, Vector{Symbol}}, observation::AbstractObservation; label::Symbol = assembleFactorName(variableorder), - timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(), + timestamp::Union{TimeDateZone, ZonedDateTime} = now_tdz(), tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}([:FACTOR]), bloblets::Bloblets = Bloblets(), multihypo::Vector{Float64} = Float64[], @@ -160,7 +160,7 @@ function FactorDFG( state::Recipestate = Recipestate(), cache = nothing; tags::Set{Symbol} = Set{Symbol}([:FACTOR]), - timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = tdz_now(), + timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = now_tdz(), solvable::Int = 1, bloblets::Bloblets = Bloblets(), blobentries::Blobentries = Blobentries(), @@ -235,7 +235,7 @@ end function FactorSummary( label::Symbol, variableorder::Union{Vector{Symbol}, Tuple}; - timestamp::TimeDateZone = tdz_now(), + timestamp::TimeDateZone = now_tdz(), tags::Set{Symbol} = Set{Symbol}(), ) return FactorSummary(label, tags, Tuple(variableorder), timestamp) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index ac0f8280..25165fad 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -128,6 +128,15 @@ end ##------------------------------------------------------------------------------ # The Variable information packed in a way that accomdates multi-lang using json. +variable_timestamp_note = """ +!!! note + This single timestamp does not represent the temporal uncertainty of non-parametric beliefs. + A single timestamp value cannot capture the distribution of temporal + information across all particles/points in a belief. For problems where time is a state variable + requiring inference (e.g., `SGal3` which includes temporal components), include time as part of + your state type rather than relying on this metadata field. +""" + """ $(TYPEDEF) Complete variable structure for a DistributedFactorGraph variable. @@ -140,12 +149,12 @@ $(TYPEDFIELDS) """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol - """Variable timestamp. + """Variable event timestamp (UTC-based) with timezone support. + $variable_timestamp_note Accessors: [`getTimestamp`](@ref)""" - timestamp::TimeDateZone = tdz_now() #NOTE changed to TimeDateZone in v0.29 + timestamp::TimeDateZone = now_tdz() #NOTE changed to TimeDateZone in v0.29 # """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)""" - # steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29 - #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime + # nstime::String = "0" #NOTE deprecated field in v0.29 """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`listTags`](@ref), [`mergeTags!`](@ref), and [`deleteTags!`](@ref)""" tags::Set{Symbol} = Set{Symbol}() @@ -174,7 +183,7 @@ function StructUtils.fielddefaults( ::Type{VariableDFG{T, P, N}}, ) where {T, P, N} return ( - timestamp = tdz_now(), + timestamp = now_tdz(), tags = Set{Symbol}(), # states = OrderedDict{Symbol, State{T, P, N}}(), bloblets = Bloblets(), @@ -213,9 +222,8 @@ function VariableDFG( label::Symbol, statetype::Union{T, Type{T}}; tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), - timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(), + timestamp::Union{TimeDateZone, ZonedDateTime} = now_tdz(), solvable::Union{Int, Base.RefValue{Int}} = Ref{Int}(1), - # steadytime::Union{Nothing, Nanosecond} = nothing, nanosecondtime = nothing, smalldata = nothing, kwargs..., @@ -285,7 +293,8 @@ $(TYPEDFIELDS) """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol - """Variable timestamp. + """Variable event timestamp. + $variable_timestamp_note Accessors: [`getTimestamp`](@ref)""" timestamp::TimeDateZone """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. diff --git a/src/serialization/PackedSerialization.jl b/src/serialization/PackedSerialization.jl index 8bbbebcc..8ccd8010 100644 --- a/src/serialization/PackedSerialization.jl +++ b/src/serialization/PackedSerialization.jl @@ -84,3 +84,38 @@ end # push!(md, kwargs...) # return md # end + +""" + DFG.@packed + +Macro annotation for DFG serialization metadata on struct fields. +Expands to `(lower = DFG.Packed, choosetype = DFG.resolvePackedType)` for use with +StructTypes.jl field annotations. + +Used to mark belief fields in factor types for serialization through +the DFG packing system. The `lower` function converts the field to a `Packed` wrapper +during serialization, and `choosetype` resolves the correct type during deserialization. + +# Usage +Use with the `&` operator in `@kwarg` or `@tags` struct definitions: + +```julia +@kwarg struct Pose2Point2Range{T} <: AbstractRelativeObservation + Z::T & DFG.@packed + partial::Tuple{Int, Int} = (1, 2) +end +``` + +This is equivalent to writing: +```julia +@kwarg struct Pose2Point2Range{T} <: AbstractRelativeObservation + Z::T & (lower = DFG.Packed, choosetype = DFG.resolvePackedType) + partial::Tuple{Int, Int} = (1, 2) +end +``` + +See also: [`Packed`](@ref), [`pack`](@ref), [`unpack`](@ref), [`resolvePackedType`](@ref) +""" +macro packed() + return esc(:(lower = DFG.Packed, choosetype = DFG.resolvePackedType)) +end diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 45bc3c39..b92c460a 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -359,17 +359,17 @@ function listNeighbors end """ $(SIGNATURES) Get a VariableDFG with a specific solver key. -In memory types still return a reference, other types returns a variable with only solveKey. +In memory types still return a reference, other types returns a variable with only stateLabel. """ -function getVariable(dfg::AbstractDFG, label::Symbol, solveKey::Symbol) - # TODO maybe change solveKey param to stateLabelFilter +function getVariable(dfg::AbstractDFG, label::Symbol, stateLabel::Symbol) + # TODO maybe change stateLabel param to stateLabelFilter # function getVariable(dfg::AbstractDFG, label::Symbol; stateLabelFilter::Union{Nothing, ...} = nothing) var = getVariable(dfg, label) - if isa(var, VariableDFG) && !haskey(var.states, solveKey) - throw(LabelNotFoundError("VariableNode", solveKey)) + if isa(var, VariableDFG) && !haskey(var.states, stateLabel) + throw(LabelNotFoundError("VariableNode", stateLabel)) elseif !isa(var, VariableDFG) - @warn "getVariable(dfg, label, solveKey) only supported for type VariableDFG." + @warn "getVariable(dfg, label, stateLabel) only supported for type VariableDFG." end return var @@ -559,7 +559,12 @@ function deepcopyGraph( graphLabel::Symbol = Symbol(getGraphLabel(sourceDFG), "_cp_$(string(uuid4())[1:6])"), kwargs..., ) where {T <: AbstractDFG} - destDFG = T(; graph = sourceDFG.graph, agent = sourceDFG.agent, graphLabel) + destDFG = T(; + solverParams = getSolverParams(sourceDFG), + graph = sourceDFG.graph, + agent = sourceDFG.agent, + graphLabel, + ) copyGraph!( destDFG, sourceDFG, diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 691520d8..202c08f9 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -11,7 +11,6 @@ function printVariable( compact::Bool = true, limit::Bool = true, skipfields::Vector{Symbol} = Symbol[], - solveKeys::Vector{Symbol} = Symbol[], ) ioc = IOContext(io, :limit => limit, :compact => compact) diff --git a/test/testBlocks.jl b/test/testBlocks.jl index c47b78d8..3b31eb32 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -242,7 +242,13 @@ function DFGVariableSCA() v1_lbl = :a v1_tags = Set([:VARIABLE, :POSE]) - testTimestamp = now(localzone()) + #test some Timestamp helpers + ts1 = DFG.Timestamp(Nanosecond(1760700359563000064), localzone()) + ts2 = DFG.Timestamp(1760700359.563000064, localzone()) + @test ts1 == ts2 + ts3 = DFG.Timestamp("2020-08-11T00:12:03.000-05:00") + ts4 = DFG.Timestamp(Val(:rata), 63732787923.0, FixedTimeZone("UTC-05:00")) + @test ts3 == ts4 # Constructors v1 = VariableDFG( v1_lbl, @@ -260,7 +266,7 @@ function DFGVariableSCA() v3 = VariableDFG( :c, State{TestVariableType2}(; label = :default); - timestamp = ZonedDateTime("2020-08-11T00:12:03.000-05:00"), + timestamp = DFG.Timestamp("2020-08-11T00:12:03.000-05:00"), ) vorphan = VariableDFG(