Merge branch 'master' of https://github.com/JuliaPOMDP/POMDPs.jl

Author: zsunberg

docs/src/api.md

@@ -72,6 +72,7 @@ These functions should return *states*, *observations*, and/or *rewards*.
 ```@docs
 gen
 initialstate
+initialobs
 ```
 
 ### [Common](@id common_api)

docs/src/generative.md

@@ -19,6 +19,8 @@ The *generative* interface consists of two functions:
 The generative interface is typically used when it is easier to return sampled states and observations rather than explicit distributions as in the [Explicit interface](@ref explicit_doc).
 This type of model is often referred to as a "black-box" model.
 
+In some special cases (e.g. reinforcement learning with [RLInterface.jl](https://github.com/JuliaPOMDP/RLInterface.jl)), an initial observation is needed before any actions are taken. In this case, the [`initialobs`](@ref) function will be used.
+
 ## The [`gen`](@ref) function
 
 The [`gen`](@ref) function has three versions differentiated by the type of the first argument.

src/POMDPs.jl

@@ -32,6 +32,7 @@ export
     # Generative model functions
     gen,
     initialstate,
+    initialobs,
 
     # Discrete Functions
     length,

src/errors.jl

@@ -3,7 +3,7 @@ struct DistributionNotImplemented <: Exception
     gen_firstarg::Type
     func::Function
     modeltype::Type
-    dep_argtypes::NamedTuple
+    dep_argtypes::AbstractVector
 end
 
 function Base.showerror(io::IO, ex::DistributionNotImplemented)
@@ -16,17 +16,20 @@ function Base.showerror(io::IO, ex::DistributionNotImplemented)
 
     i = 1
     if ex.gen_firstarg <: DDNOut
-        printstyled(io, "$i) Implement POMDPs.gen($argstring, ::AbstractRNG) to return a NamedTuple with key :$(ex.sym).\n", bold=true)
-        gen_analysis(io, ex)
+        M = ex.modeltype
+        S = statetype(M)
+        A = actiontype(M)
+        printstyled(io, "$i) Implement POMDPs.gen(::$M, ::$S, ::$A, ::AbstractRNG) to return a NamedTuple with key :$(ex.sym).\n", bold=true)
+        gen_analysis(io, ex.sym, M, [S,A])
         println(io)
         i += 1
     end
     printstyled(io, "$i) Implement POMDPs.gen(::DDNNode{:$(ex.sym)}, $argstring, ::AbstractRNG).\n",
                 bold=true)
-    Base.show_method_candidates(io, MethodError(gen, Tuple{DDNNode{ex.sym}, ex.modeltype, ex.dep_argtypes..., AbstractRNG})) # this is not exported - it may break
+    try_show_method_candidates(io, MethodError(gen, Tuple{DDNNode{ex.sym}, ex.modeltype, ex.dep_argtypes..., AbstractRNG}))
     i += 1
     printstyled(io, "\n\n$i) Implement $(ex.func)($argstring).\n", bold=true)
-    Base.show_method_candidates(io, MethodError(transition, Tuple{ex.modeltype, ex.dep_argtypes...}))
+    try_show_method_candidates(io, MethodError(ex.func, Tuple{ex.modeltype, ex.dep_argtypes...}))
 
     println(io, "\n\nThis error message uses heuristics to make recommendations for POMDPs.jl problem implementers. If it was misleading or you believe there is an inconsistency, please file an issue: https://github.com/JuliaPOMDP/POMDPs.jl/issues/new")
 end
@@ -36,63 +39,78 @@ function distribution_impl_error(sym, func, modeltype, dep_argtypes)
     acceptable = (:distribution_impl_error, nameof(func), nameof(gen), nameof(genout))
     gen_firstarg = nothing # The first argument to the `gen` call that is furthest down in the stack trace
 
-    for sf in stacktrace() # step up the stack trace
+    try
+        for sf in stacktrace() # step up the stack trace
 
-        # if it is a macro from ddn_struct.jl or gen_impl.jl it is ok
-        if sf.func === Symbol("macro expansion")
-            bn = basename(String(sf.file))
-            if !(bn in ["ddn_struct.jl", "gen_impl.jl", "none"])
-                break
-                # the call stack includes a macro from some other package
-            end
+            # if it is a macro from ddn_struct.jl or gen_impl.jl it is ok
+            if sf.func === Symbol("macro expansion")
+                bn = basename(String(sf.file))
+                if !(bn in ["ddn_struct.jl", "gen_impl.jl", "none"])
+                    break
+                    # the call stack includes a macro from some other package
+                end
 
-        # if it is not a function we know about, give up
-        elseif !(sf.func in acceptable)
-            break
+            # if it is not a function we know about, give up
+            elseif !(sf.func in acceptable)
+                break
 
-        # if it is gen, check to see if it's the DDNNode version
-        elseif sf.func === nameof(gen)
-            sig = sf.linfo.def.sig
-            if sig isa UnionAll &&
-                sig.body.parameters[1] == typeof(gen) &&
-                sig.body.parameters[2] <: Union{DDNNode, DDNOut}
-                # bingo!
-                gen_firstarg = sig.body.parameters[2]
+            # if it is gen, check to see if it's the DDNNode version
+            elseif sf.func === nameof(gen)
+                sig = sf.linfo.def.sig
+                if sig isa UnionAll &&
+                    sig.body.parameters[1] == typeof(gen) &&
+                    sig.body.parameters[2] <: Union{DDNNode, DDNOut}
+                    # bingo!
+                    gen_firstarg = sig.body.parameters[2]
+                    dep_argtypes = [sig.body.parameters[3:end-1]...]
+                end
             end
         end
+    catch ex
+        @debug("Error throwing DistributionNotImplemented error:\n$(sprint(showerror, ex))")
+        throw(MethodError(func, Tuple{modeltype, dep_argtypes...}))
     end
 
     if gen_firstarg === nothing
-        throw(MethodError(transition, Tuple{modeltype, dep_argtypes...}))
+        throw(MethodError(func, Tuple{modeltype, dep_argtypes...}))
     else
-        throw(DistributionNotImplemented(:sp, gen_firstarg, func, modeltype, dep_argtypes))
+        throw(DistributionNotImplemented(sym, gen_firstarg, func, modeltype, dep_argtypes))
     end
 end
 
-function gen_analysis(io, ex::DistributionNotImplemented)
-    argtypes = Tuple{ex.modeltype, ex.dep_argtypes..., AbstractRNG}
+function gen_analysis(io, sym::Symbol, modeltype::Type, dep_argtypes)
+    argtypes = Tuple{modeltype, dep_argtypes..., AbstractRNG}
     rts = Base.return_types(gen, argtypes)
-    @assert length(rts) > 0 # there should always be the default NamedTuple() impl.
-    if length(rts) == 1
+    if length(rts) <= 0 # there should always be the default NamedTuple() impl.
+        @debug("Error analyzing the return types for gen. Please submit an issue at https://github.com/JuliaPOMDP/POMDPs.jl/issues/new", argtypes=argtypes, rts=rts)
+    elseif length(rts) == 1
         rt = first(rts)
         if rt == typeof(NamedTuple()) && !implemented(gen, argtypes)
-            Base.show_method_candidates(io, MethodError(gen, argtypes))
+            try_show_method_candidates(io, MethodError(gen, argtypes))
             println(io)
         else
-            println(io, "\nThis method was implemented and the return type was inferred to be $rt. Is this type always a NamedTuple with key :$(ex.sym)?")
+            println(io, "\nThis method was implemented and the return type was inferred to be $rt. Is this type always a NamedTuple with key :$(sym)?")
         end
     else
         println(io, "(POMDPs.jl could not determine if this method was implemented correctly. [Base.return_types(gen, argtypes) = $(rts)])")
     end
 end
 
-transition(m, s, a) = distribution_impl_error(:sp, transition, typeof(m), (s=typeof(s), a=typeof(a)))
+function try_show_method_candidates(io, args...)
+    try
+        Base.show_method_candidates(io, args...) # this isn't exported, so it might break
+    catch ex
+        @debug("Unable to show method candidates. Please submit an issue at https://github.com/JuliaPOMDP/POMDPs.jl/issues/new.\n$(sprint(showerror, ex))")
+    end
+end
+
+transition(m, s, a) = distribution_impl_error(:sp, transition, typeof(m), [typeof(s), typeof(a)])
 function implemented(t::typeof(transition), TT::TupleType)
     m = which(t, TT)
     return m.module != POMDPs # see if this was implemented by a user elsewhere
 end
 
-observation(m, sp) = distribution_impl_error(:o, observation, typeof(m), (sp=typeof(sp),))
+observation(m, sp) = distribution_impl_error(:o, observation, typeof(m), [typeof(sp)])
 function implemented(o::typeof(observation), TT::Type{Tuple{M, SP}}) where {M<:POMDP, SP}
     m = which(o, TT)
     return m.module != POMDPs

src/generative.jl

@@ -101,19 +101,59 @@ end
         return rand(rng, d)
     end
 
+    # it is technically illegal to call this within the generated function
     if implemented(initialstate_distribution, Tuple{p})
         return impl
     else
-        req = @req initialstate_distribution(::p)
-        reqs = [(implemented(req...), req...)]
-        this = @req(initialstate(::p, ::rng))
         return quote
             try
                 $impl # trick to get the compiler to insert the right backedges
             catch
-                # TODO failed_synth_warning($this, $reqs)
                 throw(MethodError(initialstate, (p, rng)))
             end
         end
     end
 end
+
+"""
+    initialobs(m::POMDP, s, rng::AbstractRNG)
+
+Return a sampled initial observation for the problem `m` and state `s`.
+
+This function is only used in cases where the policy expects an initial observation rather than an initial belief, e.g. in a reinforcement learning setting. It is not used in a standard POMDP simulation.
+
+By default, it will fall back to `observation(m, s)`. The random number generator `rng` should be used to draw this sample (e.g. use `rand(rng)` instead of `rand()`).
+"""
+function initialobs end
+
+function implemented(f::typeof(initialobs), TT::Type)
+    if !hasmethod(f, TT)
+        return false
+    end
+    m = which(f, TT)
+    if m.module == POMDPs && !implemented(observation, Tuple{TT.parameters[1:2]...})
+        return false
+    else
+        return true
+    end
+end
+
+@generated function initialobs(m::POMDP, s, rng::AbstractRNG)
+    impl = quote
+        d = observation(m, s)
+        return rand(rng, d)
+    end
+
+    # it is technically illegal to call this within the generated function
+    if implemented(observation, Tuple{m, s})
+        return impl
+    else
+        return quote
+            try
+                $impl # trick to get the compiler to insert the right backedges
+            catch
+                throw(MethodError(initialobs, (m, s, rng)))
+            end
+        end
+    end
+end

test/test_generative.jl

@@ -2,10 +2,29 @@ import POMDPs: transition, reward, initialstate_distribution
 import POMDPs: gen
 
 struct W <: POMDP{Int, Bool, Int} end
+@test !@implemented initialstate(::W, ::typeof(Random.GLOBAL_RNG))
+@test !@implemented initialstate(::W, ::typeof(Random.GLOBAL_RNG), ::Nothing) # wrong number args
 @test_throws MethodError initialstate(W(), Random.GLOBAL_RNG)
+@test !@implemented initialobs(::W, ::Int, ::typeof(Random.GLOBAL_RNG))
+@test !@implemented initialobs(::W, ::Int, ::typeof(Random.GLOBAL_RNG), ::Nothing) # wrong number args
+@test_throws MethodError initialobs(W(), 1, Random.GLOBAL_RNG)
 @test_throws DistributionNotImplemented gen(DDNNode(:sp), W(), 1, true, Random.GLOBAL_RNG)
+try
+    gen(DDNNode(:sp), W(), 1, true, Random.GLOBAL_RNG)
+catch ex
+    str = sprint(showerror, ex)
+    @test occursin(":sp", str)
+    @test occursin("transition", str)
+end
 @test_throws DistributionNotImplemented gen(DDNOut(:sp,:r), W(), 1, true, Random.GLOBAL_RNG)
 @test_throws DistributionNotImplemented gen(DDNNode(:o), W(), 1, true, 2, Random.GLOBAL_RNG)
+try
+    gen(DDNNode(:o), W(), 1, true, 2, Random.GLOBAL_RNG)
+catch ex
+    str = sprint(showerror, ex)
+    @test occursin(":o", str)
+    @test occursin("observation", str)
+end
 @test_throws DistributionNotImplemented gen(DDNOut(:sp,:o), W(), 1, true, Random.GLOBAL_RNG)
 @test_throws DistributionNotImplemented gen(DDNOut(:sp,:o,:r), W(), 1, true, Random.GLOBAL_RNG)
 POMDPs.gen(::W, ::Int, ::Bool, ::AbstractRNG) = nothing
@@ -34,7 +53,11 @@ gen(::DDNNode{:o}, b::B, s::Int, a::Bool, sp::Int, rng::AbstractRNG) = sp
 @test @inferred gen(DDNOut(:sp,:o,:r), B(), 1, true, Random.GLOBAL_RNG) == (2, 2, -1.0)
 
 initialstate_distribution(b::B) = Int[1,2,3]
+@test @implemented initialstate(::B, ::MersenneTwister)
 @test initialstate(B(), Random.GLOBAL_RNG) in initialstate_distribution(B())
+POMDPs.observation(b::B, s::Int) = Bool[s]
+@test @implemented initialobs(::B, ::Int, ::MersenneTwister)
+@test initialobs(B(), 1, Random.GLOBAL_RNG) == 1
 
 mutable struct C <: POMDP{Nothing, Nothing, Nothing} end
 gen(::DDNNode{:sp}, c::C, s::Nothing, a::Nothing, rng::AbstractRNG) = nothing
@@ -57,6 +80,5 @@ struct GE <: MDP{Int, Int} end
 @test_throws DistributionNotImplemented gen(DDNNode(:sp), GE(), 1, 1, Random.GLOBAL_RNG)
 @test_throws DistributionNotImplemented gen(DDNOut(:sp,:r), GE(), 1, 1, Random.GLOBAL_RNG)
 POMDPs.gen(::GE, s, a, ::AbstractRNG) = (sp=s+a, r=s^2)
-@show gen(DDNOut(:sp), GE(), 1, 1, Random.GLOBAL_RNG)
 @test @inferred gen(DDNOut(:sp), GE(), 1, 1, Random.GLOBAL_RNG) == 2
 @test @inferred gen(DDNOut(:sp,:r), GE(), 1, 1, Random.GLOBAL_RNG) == (2, 1)

test/test_generative_backedges.jl

@@ -10,7 +10,11 @@ let
     @test_throws DistributionNotImplemented gen(DDNOut(:sp,:o,:r), M(), 1, 1, MersenneTwister(4))
     @test_throws DistributionNotImplemented gen(DDNOut(:sp,:r), M(), 1, 1, MersenneTwister(4))
     POMDPs.reward(::M, ::Int, ::Int, ::Int) = 0.0
-    POMDPs.gen(::DDNNode{:o}, ::M, ::Int, ::Int, ::Int, ::AbstractRNG) = `a`
+    POMDPs.gen(::DDNNode{:o}, ::M, ::Int, ::Int, ::Int, ::AbstractRNG) = 'a'
     @test gen(DDNOut(:sp,:r), M(), 1, 1, MersenneTwister(4)) == (1, 0.0)
-    @test gen(DDNOut(:sp,:o,:r), M(), 1, 1, MersenneTwister(4)) == (1, `a`, 0.0)
+    @test gen(DDNOut(:sp,:o,:r), M(), 1, 1, MersenneTwister(4)) == (1, 'a', 0.0)
+
+    @test_throws MethodError initialobs(M(), 1, MersenneTwister(4))
+    POMDPs.observation(::M, ::Int) = ['a']
+    @test initialobs(M(), 1, MersenneTwister(4)) == 'a'
 end

test/test_requirements.jl

@@ -64,6 +64,10 @@ end
 
 reqs = nothing # to check the hygeine of the macro
 println("There should be a warning about no @reqs here:")
+# 27 minutes has been spent trying to suppress this warning and automate a test for it. If you work more on it, please update this counter. The following things have been tried
+# - @test_logs (:warn, "No") @POMDP_requirements ...
+# - @capture_err @POMDP_requirements ... # From Suppressor.jl
+# - @capture_out @POMDP_requirements ... # From Suppressor.jl
 @POMDP_requirements "Warn none" begin
     1+1
 end
@@ -90,6 +94,4 @@ POMDPs.observations(p::SimplePOMDP) = [1,2,3]
 Random.rand(rng::AbstractRNG, d::SimpleDistribution) = sample(rng, d.ss, WeightVec(d.b))
 POMDPs.gen(::DDNOut{:o}, m::SimplePOMDP, s, a, rng) = 1
 
-println("There should be no warnings or requirements output below this point!\n")
-
 @test solve(CoolSolver(), SimplePOMDP())