Set compat to require SIMDPirates 0.7.10 (fixes #89), ignore :inbounds meta expr (fixes #88), and check whether functions in lowered broadcast expressions are SSA values (fixes #87).

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.6.29"
+version = "0.6.30"
 
 [deps]
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -13,7 +13,7 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 OffsetArrays = "1"
-SIMDPirates = "0.7.8"
+SIMDPirates = "0.7.10"
 SLEEFPirates = "0.4.4"
 UnPack = "0"
 VectorizationBase = "0.10"

src/broadcast.jl

@@ -169,6 +169,13 @@ function add_broadcast!(
 ) where {T<:Number}
     add_constant!(ls, bcname, elementbytes) # or replace elementbytes with sizeof(T) ? u
 end
+function add_broadcast!(
+    ls::LoopSet, ::Symbol, bcname::Symbol, loopsyms::Vector{Symbol}, ::Type{Base.RefValue{T}}, elementbytes::Int
+) where {T}
+    refextract = gensym(bcname)
+    pushpreamble!(ls, Expr(:(=), refextract, Expr(:ref, bcname)))
+    add_constant!(ls, refextract, elementbytes) # or replace elementbytes with sizeof(T) ? u
+end
 function add_broadcast!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol},
     ::Type{SubArray{T,N,A,S,B}}, elementbytes::Int

src/constructors.jl

@@ -7,7 +7,11 @@ function Base.copyto!(ls::LoopSet, q::Expr)
 end
 
 function add_ci_call!(q::Expr, f, args, syms, i, mod = nothing)
-    call = Expr(:call, f)
+    call = if f isa Core.SSAValue
+        Expr(:call, syms[f.id])
+    else
+        Expr(:call, f)
+    end
     for arg ∈ @view(args[2:end])
         if arg isa Core.SSAValue
             push!(call.args, syms[arg.id])

src/graphs.jl

@@ -244,7 +244,7 @@ end
 
 includesarray(ls::LoopSet, array::Symbol) = array ∈ ls.includedarrays
 
-function LoopSet(mod::Symbol, W = Symbol("##Wvecwidth##"), T = Symbol("Tloopeltype"))# = :LoopVectorization)
+function LoopSet(mod::Symbol, W = Symbol("##Wvecwidth##"), T = Symbol("##Tloopeltype##"))# = :LoopVectorization)
     LoopSet(
         Symbol[], [0], Loop[],
         Dict{Symbol,Operation}(),
@@ -332,6 +332,7 @@ end
 function add_block!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
     for x ∈ ex.args
         x isa Expr || continue # be that general?
+        x.head === :inbounds && continue
         push!(ls, x, elementbytes, position)
     end
 end
@@ -546,6 +547,18 @@ function Base.push!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
         add_andblock!(ls, ex, elementbytes, position)
     elseif ex.head === :||
         add_orblock!(ls, ex, elementbytes, position)
+    elseif ex.head === :local # Handle locals introduced by `@inbounds`; using `local` with `@avx` is not recomended (nor is `@inbounds`; which applies automatically regardless)
+        @assert length(ex.args) == 1 # TODO replace assert + first with "only" once support for Julia < 1.4 is dropped
+        localbody = first(ex.args)
+        @assert localbody.head === :(=)
+        @assert length(localbody.args) == 2
+        LHS = (localbody.args[1])::Symbol
+        RHS = push!(ls, (localbody.args[2]), elementbytes, position)
+        if isstore(RHS)
+            RHS
+        else
+            add_compute!(ls, LHS, :identity, [RHS], elementbytes)
+        end
     else
         throw("Don't know how to handle expression:\n$ex")
     end

src/reconstruct_loopset.jl

@@ -413,7 +413,7 @@ function _avx_loopset(OPSsv, ARFsv, AMsv, LPSYMsv, LBsv, vargs)
     )
 end
 @generated function _avx_!(::Val{UT}, ::Type{OPS}, ::Type{ARF}, ::Type{AM}, ::Type{LPSYM}, lb::LB, vargs...) where {UT, OPS, ARF, AM, LPSYM, LB}
-    1 + 1
+    # 1 + 1 # Irrelevant line you can comment out/in to force recompilation...
     ls = _avx_loopset(OPS.parameters, ARF.parameters, AM.parameters, LPSYM.parameters, LB.parameters, vargs)
     avx_body(ls, UT)
 end

src/split_loops.jl

@@ -70,8 +70,8 @@ function lower_and_split_loops(ls::LoopSet)
         remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
         ls_2 = split_loopset(ls, remaining_ops)
         order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2 = choose_order_cost(ls_2)
-        # U_1 = T_1 = U_2 = T_2 = 2
-        if cost_1 + cost_2 < cost_fused
+        U_1 = T_1 = U_2 = T_2 = 2
+        if cost_1 + cost_2 ≤ cost_fused
             ls_2_lowered = if length(remaining_ops) > 1
                 lower_and_split_loops(ls_2)
             else

test/copy.jl

@@ -37,7 +37,7 @@ using LoopVectorization, Test
     end
     function offset_copy_avx1!(A, B)
         @_avx for i=1:size(A,1), j=1:size(B,2)
-	    A[i,j+2] = B[i,j]
+	    @inbounds A[i,j+2] = B[i,j]
         end
     end
     function offset_copyavx2!(A, B)
@@ -64,7 +64,7 @@ using LoopVectorization, Test
     end
     function make23avx!(x)
         @avx for i ∈ eachindex(x)
-            x[i] = 23
+            @inbounds x[i] = 23
         end
     end
     function make23_avx!(x)
@@ -82,6 +82,7 @@ using LoopVectorization, Test
             x[i] = a
         end
     end
+    
 
     for T ∈ (Float32, Float64, Int32, Int64)
         @show T, @__LINE__
@@ -129,6 +130,11 @@ using LoopVectorization, Test
         myfillavx!(x, a);
         fill!(q2, a);
         @test x == q2
+        q2 .= 23;
+        fill!(q1, -99999); make23_avx!(q1);
+        @test q2 == q1
+        fill!(q1, -99999); make23avx!(q1);
+        @test q2 == q1
         if T <: Union{Float32,Float64}
             make2point3avx!(x)
             fill!(q2, 2.3)

test/gemm.jl

@@ -521,11 +521,12 @@
         end
     end
 
-    function twogemms!(Ab, Bb, Cb, A, B)
+    function threegemms!(Ab, Bb, Cb, A, B, C)
         M, N = size(Cb); K = size(B,1)
         @avx for m in 1:M, k in 1:K, n in 1:N
-            Ab[m,k] += Cb[m,n] * B[k,n]
-            Bb[k,n] += A[m,k] * Cb[m,n]
+            Ab[m,k] += C[m,n] * B[k,n]
+            Bb[k,n] += A[m,k] * C[m,n]
+            Cb[m,n] += A[m,k] * B[k,n]
         end
     end
     # M = 77;
@@ -639,10 +640,11 @@
             Bbit = B .> 0.5
             fill!(C, 9999.999); AmulBavx1!(C, A, Bbit)
             @test C ≈ A * Bbit
-            Ab = zero(A); Bb = zero(B);
-            twogemms!(Ab, Bb, C, A, B)
+            Ab = zero(A); Bb = zero(B); Cb = zero(C);
+            threegemms!(Ab, Bb, Cb, A, B, C)
             @test Ab ≈ C * B'
             @test Bb ≈ A' * C
+            @test Cb ≈ A * B
         end
         @time @testset "_avx $T dynamic gemm" begin
             AmulB_avx1!(C, A, B)

test/special.jl

@@ -163,13 +163,13 @@
 
         return log1p(s-1) + u
     end
-    feq = :(for i = 1:n
-            tmp = exp(x[i] - u)
-            r[i] = tmp
-            s += tmp
-            end)
-    lsfeq = LoopVectorization.LoopSet(feq);
-    # lsfeq.operations
+    # feq = :(for i = 1:n
+    #         tmp = exp(x[i] - u)
+    #         r[i] = tmp
+    #         s += tmp
+    #         end)
+    # lsfeq = LoopVectorization.LoopSet(feq);
+    # # lsfeq.operations
 
     function vpow0!(y, x)
         @avx for i ∈ eachindex(y, x)
@@ -225,6 +225,10 @@
             y[i] = x[i] ^ -5
           end);
     ls = LoopVectorization.LoopSet(q);
+    q2 = :(for i ∈ eachindex(y, x)
+            y[i] = x[i] ^ 5
+          end);
+    ls2 = LoopVectorization.LoopSet(q2);
 
     function vpow5!(y, x)
         @avx for i ∈ eachindex(y, x)