From 214b05c936ba42d59f92a4d6f39925d7a36caac6 Mon Sep 17 00:00:00 2001 From: Marco Cognetta Date: Mon, 29 Nov 2021 00:27:27 -0500 Subject: [PATCH 1/4] fuse loops --- src/layers/normalise.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 146b7dba56..9ac4eaeeac 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -30,14 +30,14 @@ The [`Dropout`](@ref) layer is what you should use in most scenarios. """ function dropout(x, p; dims=:, active::Bool=true) active || return x - y = dropout_mask(x, p, dims=dims) - return x .* y + y = rand!(similar(x, _dropout_shape(x, dims))) + @inbounds @. y = x * _dropout_kernel(y, p, 1-p) end -@adjoint function dropout(x, p; dims=:, active::Bool=true) +Flux.@adjoint function dropout(x, p; dims=:, active::Bool=true) active || return x, Δ -> (Δ, nothing) - y = dropout_mask(x, p, dims=dims) - return x .* y, Δ -> (Δ .* y, nothing) + y = rand!(similar(x, _dropout_shape(x, dims))) + return x .* _dropout_kernel.(y, p, 1-p), Δ -> (Δ .* _dropout_kernel.(y, p, 1-p), nothing) end function dropout_mask(x, p; dims=:) From 185ab40dcb51c98a5b27ca5cb77929ad48326dc1 Mon Sep 17 00:00:00 2001 From: Marco Date: Sun, 28 Nov 2021 21:41:54 -0800 Subject: [PATCH 2/4] Update normalise.jl --- src/layers/normalise.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 9ac4eaeeac..aadebdb3fe 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -34,7 +34,7 @@ function dropout(x, p; dims=:, active::Bool=true) @inbounds @. y = x * _dropout_kernel(y, p, 1-p) end -Flux.@adjoint function dropout(x, p; dims=:, active::Bool=true) +@adjoint function dropout(x, p; dims=:, active::Bool=true) active || return x, Δ -> (Δ, nothing) y = rand!(similar(x, _dropout_shape(x, dims))) return x .* _dropout_kernel.(y, p, 1-p), Δ -> (Δ .* _dropout_kernel.(y, p, 1-p), nothing) @@ -56,7 +56,7 @@ e.g. `Dropout(p; dims = 3)` will randomly zero out entire channels on WHCN input (also called 2D dropout). Does nothing to the input once [`Flux.testmode!`](@ref) is `true`. -""" +"""` mutable struct Dropout{F,D} p::F dims::D From 1242c2015a1806394537713d2edc7c0a96c390ae Mon Sep 17 00:00:00 2001 From: Marco Date: Sun, 28 Nov 2021 21:49:03 -0800 Subject: [PATCH 3/4] Update normalise.jl --- src/layers/normalise.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index aadebdb3fe..2fce23307b 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -31,7 +31,7 @@ The [`Dropout`](@ref) layer is what you should use in most scenarios. function dropout(x, p; dims=:, active::Bool=true) active || return x y = rand!(similar(x, _dropout_shape(x, dims))) - @inbounds @. y = x * _dropout_kernel(y, p, 1-p) + @. y = x * _dropout_kernel(y, p, 1-p) end @adjoint function dropout(x, p; dims=:, active::Bool=true) @@ -56,7 +56,7 @@ e.g. `Dropout(p; dims = 3)` will randomly zero out entire channels on WHCN input (also called 2D dropout). Does nothing to the input once [`Flux.testmode!`](@ref) is `true`. -"""` +""" mutable struct Dropout{F,D} p::F dims::D From 64d554f09bcd971ccda4e1aa62b2181606a0b1bd Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 29 Nov 2021 12:08:37 +0530 Subject: [PATCH 4/4] Update src/layers/normalise.jl Co-authored-by: Brian Chen --- src/layers/normalise.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 2fce23307b..73b625c616 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -31,7 +31,7 @@ The [`Dropout`](@ref) layer is what you should use in most scenarios. function dropout(x, p; dims=:, active::Bool=true) active || return x y = rand!(similar(x, _dropout_shape(x, dims))) - @. y = x * _dropout_kernel(y, p, 1-p) + x .* _dropout_kernel.(y, p, 1-p) end @adjoint function dropout(x, p; dims=:, active::Bool=true)