Skip to content

Accurate derivative of tanh for large |x| #874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
devmotion wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Accurate derivative of tanh for large |x| #874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion src/dual.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ Base.float(d::Dual) = convert(float(typeof(d)), d)
###################################

for (M, f, arity) in DiffRules.diffrules(filter_modules = nothing)
if (M, f) in ((:Base, :^), (:NaNMath, :pow), (:Base, :/), (:Base, :+), (:Base, :-), (:Base, :sin), (:Base, :cos))
if (M, f) in ((:Base, :^), (:NaNMath, :pow), (:Base, :/), (:Base, :+), (:Base, :-), (:Base, :sin), (:Base, :cos), (:Base, :tanh))
continue # Skip methods which we define elsewhere.
elseif !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f))
continue # Skip rules for methods not defined in the current scope
Expand Down Expand Up @@ -721,6 +721,18 @@ end
return (Dual{T}(sd, cd * partials(d)), Dual{T}(cd, -sd * partials(d)))
end

# tanh #
#------#

# `1 - tanh(x)^2` reuses the primal value but suffers from catastrophic cancellation for
# large `|x|` (JuliaDiff/DiffRules.jl#102), so use `sech(x)^2` there
@inline function Base.tanh(d::Dual{T}) where T
x = value(d)
val = tanh(x)
deriv = abs(x) < 1 ? 1 - val^2 : sech(x)^2
return Dual{T}(val, deriv * partials(d))
end

# sincospi #
#----------#

Expand Down
22 changes: 21 additions & 1 deletion test/DualTest.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,12 @@ ForwardDiff.:≺(::Type{OuterTestTag}, ::Type{TestTag}) = false
if actualval isa Real
@test dx isa Dual{TestTag}
@test value(dx) == actualval
@test partials(dx, 1) == $deriv
if $(M === :Base && f === :tanh)
# defined by hand with an equivalent formula that is faster for |x| < 1
@test partials(dx, 1) ≈ $deriv
else
@test partials(dx, 1) == $deriv
end
else
@test dx isa Complex{<:Dual{TestTag}}
@test value(real(dx)) == real(actualval)
Expand Down Expand Up @@ -775,4 +780,19 @@ end
end
end

@testset "tanh: no catastrophic cancellation for large |x|" begin # JuliaDiff/DiffRules.jl#102
for T in (Float32, Float64), x in (0.5, -0.99, 1.0, -5.0, 10.0, -20.0, 30.0, 300.0)
x = T(x)
expected = T(setprecision(() -> sech(big(x))^2, BigFloat, 256))
d = tanh(Dual{TestTag}(x, one(x)))
@test value(d) === tanh(x)
@test partials(d, 1) isa T
@test partials(d, 1) ≈ expected rtol = 4 * eps(T)
# nested duals take the same branch
dd = tanh(Dual{TestTag}(Dual{TestTag}(x, one(x)), one(x)))
@test value(value(dd)) === tanh(x)
@test value(partials(dd, 1)) ≈ expected rtol = 4 * eps(T)
end
end

end # module
Loading