|
| 1 | +function LinearAlgebra.mul!( |
| 2 | + @nospecialize(C::TracedRArray{T1,1}), |
| 3 | + @nospecialize(A::AnyTracedRArray{T2,2}), |
| 4 | + @nospecialize(B::AnyTracedRArray{T3,1}), |
| 5 | + α::Number=true, |
| 6 | + β::Number=false, |
| 7 | +) where {T1,T2,T3} |
| 8 | + # TODO: The reshape operations are not getting optimized, we should directly call dot_general |
| 9 | + rC = reshape(C, :, 1) |
| 10 | + LinearAlgebra.mul!(rC, A, reshape(B, :, 1), α, β) |
| 11 | + C.mlir_data = get_mlir_data(vec(rC)) |
| 12 | + return C |
| 13 | +end |
| 14 | + |
| 15 | +function LinearAlgebra.mul!( |
| 16 | + @nospecialize(C::TracedRArray{T1,2}), |
| 17 | + @nospecialize(A::AnyTracedRArray{T2,2}), |
| 18 | + @nospecialize(B::AnyTracedRArray{T3,1}), |
| 19 | + α::Number=true, |
| 20 | + β::Number=false, |
| 21 | +) where {T1,T2,T3} |
| 22 | + LinearAlgebra.mul!(C, A, reshape(B, :, 1), α, β) |
| 23 | + return C |
| 24 | +end |
| 25 | + |
| 26 | +function LinearAlgebra.mul!( |
| 27 | + @nospecialize(C::TracedRArray{T1,2}), |
| 28 | + @nospecialize(A::AnyTracedRArray{T2,2}), |
| 29 | + @nospecialize(B::AnyTracedRArray{T3,2}), |
| 30 | + α::Number=true, |
| 31 | + β::Number=false, |
| 32 | +) where {T1,T2,T3} |
| 33 | + if size(C) != (size(A, 1), size(B, 2)) |
| 34 | + throw( |
| 35 | + DimensionMismatch( |
| 36 | + "C has size $(size(C)), A has size $(size(A)), B has size $(size(B))" |
| 37 | + ), |
| 38 | + ) |
| 39 | + end |
| 40 | + if size(A, 2) != size(B, 1) |
| 41 | + throw(DimensionMismatch("A has size $(size(A)), B has size $(size(B))")) |
| 42 | + end |
| 43 | + resty = MLIR.IR.TensorType(size(C), MLIR.IR.Type(T1)) |
| 44 | + dot_dimension_numbers = MLIR.API.stablehloDotDimensionNumbersGet( |
| 45 | + MLIR.IR.context(), 0, [], 0, [], 1, [1], 1, [0] |
| 46 | + ) |
| 47 | + prec = MLIR.IR.Attribute( |
| 48 | + MLIR.API.stablehloPrecisionAttrGet(MLIR.IR.context(), "DEFAULT") |
| 49 | + ) |
| 50 | + precar = MLIR.IR.Attribute([prec, prec]) |
| 51 | + res = MLIR.IR.result( |
| 52 | + MLIR.Dialects.stablehlo.dot_general( |
| 53 | + get_mlir_data(A), |
| 54 | + get_mlir_data(B); |
| 55 | + result_0=resty, |
| 56 | + dot_dimension_numbers=dot_dimension_numbers, |
| 57 | + precision_config=precar, |
| 58 | + ), |
| 59 | + 1, |
| 60 | + ) |
| 61 | + if iszero(β) |
| 62 | + if isone(α) |
| 63 | + C.mlir_data = res |
| 64 | + else |
| 65 | + C.mlir_data = MLIR.IR.result( |
| 66 | + MLIR.Dialects.stablehlo.multiply( |
| 67 | + res, broadcast_to_size(T1(α), size(C)).mlir_data |
| 68 | + ), |
| 69 | + 1, |
| 70 | + ) |
| 71 | + end |
| 72 | + else |
| 73 | + α_res = MLIR.IR.result( |
| 74 | + MLIR.Dialects.stablehlo.multiply( |
| 75 | + res, broadcast_to_size(T1(α), size(C)).mlir_data |
| 76 | + ), |
| 77 | + 1, |
| 78 | + ) |
| 79 | + β_C = MLIR.IR.result( |
| 80 | + MLIR.Dialects.stablehlo.multiply( |
| 81 | + C.mlir_data, broadcast_to_size(T1(β), size(C)).mlir_data |
| 82 | + ), |
| 83 | + 1, |
| 84 | + ) |
| 85 | + C.mlir_data = MLIR.IR.result(MLIR.Dialects.stablehlo.add(α_res, β_C), 1) |
| 86 | + end |
| 87 | + return C |
| 88 | +end |
| 89 | + |
| 90 | +function LinearAlgebra.triu!(@nospecialize(X::TracedRArray{T,2}), k::Integer) where {T} |
| 91 | + iota_1 = Ops.iota(Int64, [size(X)...]; iota_dimension=1) |
| 92 | + iota_2 = Ops.subtract( |
| 93 | + Ops.iota(Int64, [size(X)...]; iota_dimension=2), broadcast_to_size(k, size(X)) |
| 94 | + ) |
| 95 | + idxs = Ops.compare(iota_1, iota_2; comparison_direction="LE") |
| 96 | + X.mlir_data = Ops.select(idxs, X, zero(X)).mlir_data |
| 97 | + return X |
| 98 | +end |
| 99 | + |
| 100 | +function LinearAlgebra.tril!(@nospecialize(X::TracedRArray{T,2}), k::Integer) where {T} |
| 101 | + iota_1 = Ops.iota(Int64, [size(X)...]; iota_dimension=1) |
| 102 | + iota_2 = Ops.subtract( |
| 103 | + Ops.iota(Int64, [size(X)...]; iota_dimension=2), broadcast_to_size(k, size(X)) |
| 104 | + ) |
| 105 | + idxs = Ops.compare(iota_1, iota_2; comparison_direction="GE") |
| 106 | + X.mlir_data = Ops.select(idxs, X, zero(X)).mlir_data |
| 107 | + return X |
| 108 | +end |
0 commit comments