From 3bda87d61858cc890ac19ebfc8f37b83417142d7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 13 Sep 2024 17:55:58 -0700 Subject: [PATCH 01/71] Project.toml: update Symbolics deps --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 9edeb4536..b038c3364 100644 --- a/Project.toml +++ b/Project.toml @@ -36,8 +36,8 @@ NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" StatsBase = "0.33, 0.34" -Symbolics = "4, 5" -SymbolicUtils = "1.4 - 1.5" +Symbolics = "4, 5, 6" +SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 70a4e1f58f56651450fe57d3ed3ba8a75410e76d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 08:36:18 -0700 Subject: [PATCH 02/71] tests/examples: import -> using no declarations, so import is not required --- test/examples/multigroup/build_models.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 4b5afd58e..3f29a6898 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -1,3 +1,5 @@ +const SEM = StructuralEquationModels + ############################################################################################ # ML estimation ############################################################################################ From 56a1b0426461a7ec4767f4a73ddfd17f32b575e4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Nov 2024 11:28:41 -0800 Subject: [PATCH 03/71] add ParamsArray replaces RAMMatrices indices and constants vectors with dedicated class that incapsulate this logic, resulting in overall cleaner interface A_ind, S_ind, M_ind become ParamsArray F_ind becomes SparseMatrixCSC parameters.jl is not longer required and is removed --- src/StructuralEquationModels.jl | 2 +- src/additional_functions/parameters.jl | 137 ------- src/additional_functions/params_array.jl | 204 ++++++++++ .../start_val/start_fabin3.jl | 152 ++++--- .../start_val/start_simple.jl | 20 +- src/frontend/specification/RAMMatrices.jl | 381 ++++++++---------- src/imply/RAM/generic.jl | 61 +-- src/imply/RAM/symbolic.jl | 21 +- 8 files changed, 486 insertions(+), 492 deletions(-) delete mode 100644 src/additional_functions/parameters.jl create mode 100644 src/additional_functions/params_array.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 944542379..6172af1ea 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -26,6 +26,7 @@ include("objective_gradient_hessian.jl") # helper objects and functions include("additional_functions/commutation_matrix.jl") +include("additional_functions/params_array.jl") # fitted objects include("frontend/fit/SemFit.jl") @@ -69,7 +70,6 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/parameters.jl") include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_partable.jl") diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl deleted file mode 100644 index d6e8eb535..000000000 --- a/src/additional_functions/parameters.jl +++ /dev/null @@ -1,137 +0,0 @@ -# fill A, S, and M matrices with the parameter values according to the parameters map -function fill_A_S_M!( - A::AbstractMatrix, - S::AbstractMatrix, - M::Union{AbstractVector, Nothing}, - A_indices::AbstractArrayParamsMap, - S_indices::AbstractArrayParamsMap, - M_indices::Union{AbstractArrayParamsMap, Nothing}, - params::AbstractVector, -) - @inbounds for (iA, iS, par) in zip(A_indices, S_indices, params) - for index_A in iA - A[index_A] = par - end - - for index_S in iS - S[index_S] = par - end - end - - if !isnothing(M) - @inbounds for (iM, par) in zip(M_indices, params) - for index_M in iM - M[index_M] = par - end - end - end -end - -# build the map from the index of the parameter to the linear indices -# of this parameter occurences in M -# returns ArrayParamsMap object -function array_params_map(params::AbstractVector, M::AbstractArray) - params_index = Dict(param => i for (i, param) in enumerate(params)) - T = Base.eltype(eachindex(M)) - res = [Vector{T}() for _ in eachindex(params)] - for (i, val) in enumerate(M) - par_ind = get(params_index, val, nothing) - if !isnothing(par_ind) - push!(res[par_ind], i) - end - end - return res -end - -function eachindex_lower(M; linear_indices = false, kwargs...) - indices = CartesianIndices(M) - indices = filter(x -> (x[1] >= x[2]), indices) - - if linear_indices - indices = cartesian2linear(indices, M) - end - - return indices -end - -function cartesian2linear(ind_cart, dims) - ind_lin = LinearIndices(dims)[ind_cart] - return ind_lin -end - -function linear2cartesian(ind_lin, dims) - ind_cart = CartesianIndices(dims)[ind_lin] - return ind_cart -end - -function set_constants!(M, M_pre) - for index in eachindex(M) - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - M_pre[index] = δ - end - end -end - -function check_constants(M) - for index in eachindex(M) - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - return true - end - end - - return false -end - -# construct length(M)×length(parameters) sparse matrix of 1s at the positions, -# where the corresponding parameter occurs in the M matrix -function matrix_gradient(M_indices::ArrayParamsMap, M_length::Integer) - rowval = reduce(vcat, M_indices) - colptr = - pushfirst!(accumulate((ptr, M_ind) -> ptr + length(M_ind), M_indices, init = 1), 1) - return SparseMatrixCSC( - M_length, - length(M_indices), - colptr, - rowval, - ones(length(rowval)), - ) -end - -# fill M with parameters -function fill_matrix!( - M::AbstractMatrix, - M_indices::AbstractArrayParamsMap, - params::AbstractVector, -) - for (iM, par) in zip(M_indices, params) - for index_M in iM - M[index_M] = par - end - end - return M -end - -# range of parameters that are referenced in the matrix -function param_range(mtx_indices::AbstractArrayParamsMap) - first_i = findfirst(!isempty, mtx_indices) - last_i = findlast(!isempty, mtx_indices) - - if !isnothing(first_i) && !isnothing(last_i) - for i in first_i:last_i - if isempty(mtx_indices[i]) - # TODO show which parameter is missing in which matrix - throw( - ErrorException( - "Your parameter vector is not partitioned into directed and undirected effects", - ), - ) - end - end - end - - return first_i:last_i -end diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl new file mode 100644 index 000000000..f20a6518b --- /dev/null +++ b/src/additional_functions/params_array.jl @@ -0,0 +1,204 @@ +""" +Array with partially parameterized elements. +""" +struct ParamsArray{T, N} <: AbstractArray{T, N} + linear_indices::Vector{Int} + param_ptr::Vector{Int} + constants::Vector{Pair{Int, T}} + size::NTuple{N, Int} +end + +ParamsVector{T} = ParamsArray{T, 1} +ParamsMatrix{T} = ParamsArray{T, 2} + +function ParamsArray{T, N}( + params_map::AbstractVector{<:AbstractVector{Int}}, + constants::Vector{Pair{Int, T}}, + size::NTuple{N, Int}, +) where {T, N} + params_ptr = + pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) + return ParamsArray{T, N}( + reduce(vcat, params_map, init = Vector{Int}()), + params_ptr, + constants, + size, + ) +end + +function ParamsArray{T, N}( + arr::AbstractArray{<:Any, N}, + params::AbstractVector{Symbol}; + skip_zeros::Bool = true, +) where {T, N} + params_index = Dict(param => i for (i, param) in enumerate(params)) + constants = Vector{Pair{Int, T}}() + params_map = [Vector{Int}() for _ in eachindex(params)] + arr_ixs = CartesianIndices(arr) + for (i, val) in pairs(vec(arr)) + ismissing(val) && continue + if isa(val, Number) + (skip_zeros && iszero(val)) || push!(constants, i => val) + else + par_ind = get(params_index, val, nothing) + if !isnothing(par_ind) + push!(params_map[par_ind], i) + else + throw(KeyError("Unrecognized parameter $val at position $(arr_ixs[i])")) + end + end + end + return ParamsArray{T, N}(params_map, constants, size(arr)) +end + +ParamsArray{T}( + arr::AbstractArray{<:Any, N}, + params::AbstractVector{Symbol}; + kwargs..., +) where {T, N} = ParamsArray{T, N}(arr, params; kwargs...) + +nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 + +Base.size(arr::ParamsArray) = arr.size +Base.size(arr::ParamsArray, i::Integer) = arr.size[i] + +Base.:(==)(a::ParamsArray, b::ParamsArray) = return eltype(a) == eltype(b) && + size(a) == size(b) && + a.constants == b.constants && + a.param_ptr == b.param_ptr && + a.linear_indices == b.linear_indices + +# the range of arr.param_ptr indices that correspond to i-th parameter +param_occurences_range(arr::ParamsArray, i::Integer) = + arr.param_ptr[i]:(arr.param_ptr[i+1]-1) + +""" + param_occurences(arr::ParamsArray, i::Integer) + +Get the linear indices of the elements in `arr` that correspond to the +`i`-th parameter. +""" +param_occurences(arr::ParamsArray, i::Integer) = + view(arr.linear_indices, arr.param_ptr[i]:(arr.param_ptr[i+1]-1)) + +""" + materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false) + +Materialize the parameterized array `src` into `dest` by substituting the parameter +references with the parameter values from `param_values`. +""" +function materialize!( + dest::AbstractArray{<:Any, N}, + src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false, +) where {N} + size(dest) == size(src) || throw( + DimensionMismatch( + "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", + ), + ) + nparams(src) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + ), + ) + Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) + set_zeros && fill!(dest, zero(Z)) + if set_constants + @inbounds for (i, val) in src.constants + dest[i] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest[src.linear_indices[j]] = val + end + end + return dest +end + +""" + materialize([T], src::ParamsArray{<:Any, N}, + param_values::AbstractVector{T}) where T + +Materialize the parameterized array `src` into a new array of type `T` +by substituting the parameter references with the parameter values from `param_values`. +""" +materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where {T} = + materialize!(similar(arr, T), arr, param_values, set_constants = true, set_zeros = true) + +materialize(arr::ParamsArray, param_values::AbstractVector{T}) where {T} = + materialize(Union{T, eltype(arr)}, arr, param_values) + +function sparse_materialize( + ::Type{T}, + arr::ParamsMatrix, + param_values::AbstractVector, +) where {T} + nparams(arr) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))", + ), + ) + # constant values in sparse matrix + cvals = [T(v) for (_, v) in arr.constants] + # parameter values in sparse matrix + parvals = Vector{T}(undef, length(arr.linear_indices)) + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(arr, i) + parvals[j] = val + end + end + nzixs = [first.(arr.constants); arr.linear_indices] + ixorder = sortperm(nzixs) + nzixs = nzixs[ixorder] + nzvals = [cvals; parvals][ixorder] + arr_ixs = CartesianIndices(size(arr)) + return sparse( + [arr_ixs[i][1] for i in nzixs], + [arr_ixs[i][2] for i in nzixs], + nzvals, + size(arr)..., + ) +end + +sparse_materialize(arr::ParamsArray, params::AbstractVector{T}) where {T} = + sparse_materialize(Union{T, eltype(arr)}, arr, params) + +# construct length(M)×length(params) sparse matrix of 1s at the positions, +# where the corresponding parameter occurs in the arr +sparse_gradient(::Type{T}, arr::ParamsArray) where {T} = SparseMatrixCSC( + length(arr), + nparams(arr), + arr.param_ptr, + arr.linear_indices, + ones(T, length(arr.linear_indices)), +) + +sparse_gradient(arr::ParamsArray{T}) where {T} = sparse_gradient(T, arr) + +# range of parameters that are referenced in the matrix +function params_range(arr::ParamsArray; allow_gaps::Bool = false) + first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + + if !allow_gaps && !isnothing(first_i) && !isnothing(last_i) + for i in first_i:last_i + if isempty(param_occurences_range(arr, i)) + # TODO show which parameter is missing in which matrix + throw( + ErrorException( + "Parameter vector is not partitioned into directed and undirected effects", + ), + ) + end + end + end + + return first_i:last_i +end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 081af3ba1..9d692437e 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -31,23 +31,20 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F, M, n_par = ram_matrices.A, + ram_matrices.S, + ram_matrices.F, + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - n_latent = nlatent_vars(ram_matrices) - - C_indices = CartesianIndices((n_var, n_var)) + F_var2obs = Dict( + i => F.rowval[F.colptr[i]] for i in axes(F, 2) if isobserved_var(ram_matrices, i) + ) + @assert length(F_var2obs) == size(F, 1) # check in which matrix each parameter appears - indices = Vector{CartesianIndex{2}}(undef, n_par) - #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] @@ -65,26 +62,53 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end =# # set undirected parameters in S - for (i, S_ind) in enumerate(S_ind) - for c_ind in C_indices[S_ind] - (c_ind[1] == c_ind[2]) || continue # covariances stay 0 - pos = searchsortedfirst(F_ind, c_ind[1]) - start_val[i] = - (pos <= length(F_ind)) && (F_ind[pos] == c_ind[1]) ? Σ[pos, pos] / 2 : 0.05 - break # i-th parameter initialized + S_indices = CartesianIndices(S) + for j in 1:nparams(S) + for lin_ind in param_occurences(S, j) + to, from = Tuple(S_indices[lin_ind]) + if (to == from) # covariances start with 0 + # half of observed variance for observed, 0.05 for latent + obs = get(F_var2obs, to, nothing) + start_val[j] = !isnothing(obs) ? Σ[obs, obs] / 2 : 0.05 + break # j-th parameter initialized + end end end # set loadings - constants = ram_matrices.constants - A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] + A_indices = CartesianIndices(A) # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) - function calculate_lambda( - ref::Integer, - indicator::Integer, - indicators::AbstractVector{<:Integer}, - ) + # collect latent variable indicators in A + # maps latent parameter to the vector of dependent vars + # the 2nd index in the pair specified the parameter index, + # 0 if no parameter (constant), -1 if constant=1 + var2indicators = Dict{Int, Vector{Pair{Int, Int}}}() + for j in 1:nparams(A) + for lin_ind in param_occurences(A, j) + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => j) + end + end + end + + for (lin_ind, val) in A.constants + iszero(val) && continue # only non-zero loadings + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => ifelse(isone(val), -1, 0)) # no parameter associated, -1 = reference, 0 = indicator + end + end + + # calculate starting values for parameters of latent regression vars + function calculate_lambda(ref::Integer, indicator::Integer, indicators::AbstractVector) instruments = filter(i -> (i != ref) && (i != indicator), indicators) if length(instruments) == 1 s13 = Σ[ref, instruments[1]] @@ -99,61 +123,33 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end end - for i in setdiff(1:n_var, F_ind) - reference = Int64[] - indicators = Int64[] - indicator2parampos = Dict{Int, Int}() - - for (j, Aj_ind_c) in enumerate(A_ind_c) - for ind_c in Aj_ind_c - (ind_c[2] == i) || continue - ind_pos = searchsortedfirst(F_ind, ind_c[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ind_c[1]) - push!(indicators, ind_pos) - indicator2parampos[ind_pos] = j - end - end - end - - for ram_const in constants - if (ram_const.matrix == :A) && (ram_const.index[2] == i) - ind_pos = searchsortedfirst(F_ind, ram_const.index[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ram_const.index[1]) - if isone(ram_const.value) - push!(reference, ind_pos) - else - push!(indicators, ind_pos) - # no parameter associated - end - end - end - end - + for (i, indicators) in pairs(var2indicators) + reference = [obs for (obs, param) in indicators if param == -1] + indicator_obs = first.(indicators) # is there at least one reference indicator? if length(reference) > 0 - if (length(reference) > 1) && isempty(indicator2parampos) # don't warn if entire column is fixed + if (length(reference) > 1) && any(((obs, param),) -> param > 0, indicators) # don't warn if entire column is fixed @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" end ref = reference[1] - for (j, indicator) in enumerate(indicators) - if (indicator != ref) && - (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = calculate_lambda(ref, indicator, indicators) + for (indicator, param) in indicators + if (indicator != ref) && (param > 0) + start_val[param] = calculate_lambda(ref, indicator, indicator_obs) end end # no reference indicator: - elseif length(indicators) > 0 - ref = indicators[1] - λ = Vector{Float64}(undef, length(indicators)) + else + ref = indicator_obs[1] + λ = Vector{Float64}(undef, length(indicator_obs)) λ[1] = 1.0 - for (j, indicator) in enumerate(indicators) + for (j, indicator) in enumerate(indicator_obs) if indicator != ref - λ[j] = calculate_lambda(ref, indicator, indicators) + λ[j] = calculate_lambda(ref, indicator, indicator_obs) end end - Σ_λ = Σ[indicators, indicators] + Σ_λ = Σ[indicator_obs, indicator_obs] l₂ = sum(abs2, λ) D = λ * λ' ./ l₂ θ = (I - D .^ 2) \ (diag(Σ_λ - D * Σ_λ * D)) @@ -164,24 +160,22 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) λ .*= sign(Ψ) * sqrt(abs(Ψ)) - for (j, indicator) in enumerate(indicators) - if (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = λ[j] + for (j, (_, param)) in enumerate(indicators) + if param > 0 + start_val[param] = λ[j] end end - else - @warn "No scaling indicators for $(ram_matrices.colnames[i])" end end - # set means - if !isnothing(M_ind) - for (i, M_ind) in enumerate(M_ind) - if length(M_ind) != 0 - ind = M_ind[1] - pos = searchsortedfirst(F_ind, ind[1]) - if (pos <= length(F_ind)) && (F_ind[pos] == ind[1]) - start_val[i] = μ[pos] + if !isnothing(M) + # set starting values of the observed means + for j in 1:nparams(M) + M_ind = param_occurences(M, j) + if !isempty(M_ind) + obs = get(F_var2obs, M_ind[1], nothing) + if !isnothing(obs) + start_val[j] = μ[obs] end # latent means stay 0 end end diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 3b29ec178..1f73a3583 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -62,10 +62,10 @@ function start_simple( start_means = 0.0, kwargs..., ) - A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F_ind, M, n_par = ram_matrices.A, + ram_matrices.S, + observed_var_indices(ram_matrices), + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) @@ -75,9 +75,11 @@ function start_simple( C_indices = CartesianIndices((n_var, n_var)) for i in 1:n_par - if length(S_ind[i]) != 0 + Si_ind = param_occurences(S, i) + Ai_ind = param_occurences(A, i) + if length(Si_ind) != 0 # use the first occurence of the parameter to determine starting value - c_ind = C_indices[S_ind[i][1]] + c_ind = C_indices[Si_ind[1]] if c_ind[1] == c_ind[2] if c_ind[1] ∈ F_ind start_val[i] = start_variances_observed @@ -95,14 +97,14 @@ function start_simple( start_val[i] = start_covariances_obs_lat end end - elseif length(A_ind[i]) != 0 - c_ind = C_indices[A_ind[i][1]] + elseif length(Ai_ind) != 0 + c_ind = C_indices[Ai_ind[1]] if (c_ind[1] ∈ F_ind) & !(c_ind[2] ∈ F_ind) start_val[i] = start_loadings else start_val[i] = start_regressions end - elseif !isnothing(M_ind) && (length(M_ind[i]) != 0) + elseif !isnothing(M) && (length(param_occurences(M, i)) != 0) start_val[i] = start_means end end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 6ba6be3d0..b140ae026 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -1,82 +1,56 @@ -############################################################################################ -### Constants -############################################################################################ - -struct RAMConstant - matrix::Symbol - index::Union{Int, CartesianIndex{2}} - value::Any -end - -function Base.:(==)(c1::RAMConstant, c2::RAMConstant) - res = ((c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value)) - return res -end - -function append_RAMConstants!( - constants::AbstractVector{RAMConstant}, - mtx_name::Symbol, - mtx::AbstractArray; - skip_zeros::Bool = true, -) - for (index, val) in pairs(mtx) - if isa(val, Number) && !(skip_zeros && iszero(val)) - push!(constants, RAMConstant(mtx_name, index, val)) - end - end - return constants -end - -function set_RAMConstant!(A, S, M, rc::RAMConstant) - if rc.matrix == :A - A[rc.index] = rc.value - elseif rc.matrix == :S - S[rc.index] = rc.value - S[rc.index[2], rc.index[1]] = rc.value # symmetric - elseif rc.matrix == :M - M[rc.index] = rc.value - end -end - -function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) - for rc in rc_vec - set_RAMConstant!(A, S, M, rc) - end -end ############################################################################################ ### Type ############################################################################################ -# map from parameter index to linear indices of matrix/vector positions where it occurs -AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} -ArrayParamsMap = Vector{Vector{Int}} - struct RAMMatrices <: SemSpecification - A_ind::ArrayParamsMap - S_ind::ArrayParamsMap - F_ind::Vector{Int} - M_ind::Union{ArrayParamsMap, Nothing} + A::ParamsMatrix{Float64} + S::ParamsMatrix{Float64} + F::SparseMatrixCSC{Float64} + M::Union{ParamsVector{Float64}, Nothing} params::Vector{Symbol} - colnames::Union{Vector{Symbol}, Nothing} - constants::Vector{RAMConstant} - size_F::Tuple{Int, Int} + colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) end -nparams(ram::RAMMatrices) = length(ram.A_ind) - -nvars(ram::RAMMatrices) = ram.size_F[2] -nobserved_vars(ram::RAMMatrices) = ram.size_F[1] +nparams(ram::RAMMatrices) = nparams(ram.A) +nvars(ram::RAMMatrices) = size(ram.F, 2) +nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) vars(ram::RAMMatrices) = ram.colnames +isobserved_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] > ram.F.colptr[i] +islatent_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] == ram.F.colptr[i] + +# indices of observed variables in the order as they appear in ram.F rows +function observed_var_indices(ram::RAMMatrices) + obs_inds = Vector{Int}(undef, nobserved_vars(ram)) + @inbounds for i in 1:nvars(ram) + colptr = ram.F.colptr[i] + if ram.F.colptr[i+1] > colptr # is observed + obs_inds[ram.F.rowval[colptr]] = i + end + end + return obs_inds +end + +latent_var_indices(ram::RAMMatrices) = + [i for i in axes(ram.F, 2) if islatent_var(ram, i)] + +# observed variables in the order as they appear in ram.F rows function observed_vars(ram::RAMMatrices) if isnothing(ram.colnames) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, ram.F_ind) + obs_vars = Vector{Symbol}(undef, nobserved_vars(ram)) + @inbounds for (i, v) in enumerate(vars(ram)) + colptr = ram.F.colptr[i] + if ram.F.colptr[i+1] > colptr # is observed + obs_vars[ram.F.rowval[colptr]] = v + end + end + return obs_vars end end @@ -85,7 +59,7 @@ function latent_vars(ram::RAMMatrices) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, setdiff(eachindex(ram.colnames), ram.F_ind)) + return [col for (i, col) in enumerate(ram.colnames) if islatent_var(ram, i)] end end @@ -128,27 +102,16 @@ function RAMMatrices(; ), ) end - check_params(params, nothing) - A_indices = array_params_map(params, A) - S_indices = array_params_map(params, S) - M_indices = !isnothing(M) ? array_params_map(params, M) : nothing - F_indices = [i for (i, col) in zip(axes(F, 2), eachcol(F)) if any(isone, col)] - constants = Vector{RAMConstant}() - append_RAMConstants!(constants, :A, A) - append_RAMConstants!(constants, :S, S) - isnothing(M) || append_RAMConstants!(constants, :M, M) - return RAMMatrices( - A_indices, - S_indices, - F_indices, - M_indices, - params, - colnames, - constants, - size(F), - ) + A = ParamsMatrix{Float64}(A, params) + S = ParamsMatrix{Float64}(S, params) + M = !isnothing(M) ? ParamsVector{Float64}(M, params) : nothing + spF = sparse(F) + if any(!isone, spF.nzval) + throw(ArgumentError("F should contain only 0s and 1s")) + end + return RAMMatrices(A, S, F, M, params, colnames) end ############################################################################################ @@ -165,83 +128,102 @@ function RAMMatrices( n_observed = length(partable.observed_vars) n_latent = length(partable.latent_vars) - n_node = n_observed + n_latent - - # F indices - F_ind = - length(partable.sorted_vars) != 0 ? - findall(∈(Set(partable.observed_vars)), partable.sorted_vars) : 1:n_observed - - # indices of the colnames - colnames = - length(partable.sorted_vars) != 0 ? copy(partable.sorted_vars) : - [ - partable.observed_vars - partable.latent_vars - ] - col_indices = Dict(col => i for (i, col) in enumerate(colnames)) + n_vars = n_observed + n_latent + + if length(partable.sorted_vars) != 0 + @assert length(partable.sorted_vars) == nvars(partable) + vars_sorted = copy(partable.sorted_vars) + else + vars_sorted = [partable.observed_vars + partable.latent_vars] + end + + # indices of the vars (A/S/M rows or columns) + vars_index = Dict(col => i for (i, col) in enumerate(vars_sorted)) # fill Matrices # known_labels = Dict{Symbol, Int64}() - A_ind = [Vector{Int64}() for _ in 1:length(params)] - S_ind = [Vector{Int64}() for _ in 1:length(params)] - + T = nonmissingtype(eltype(partable.columns[:value_fixed])) + A_inds = [Vector{Int64}() for _ in 1:length(params)] + A_lin_ixs = LinearIndices((n_vars, n_vars)) + S_inds = [Vector{Int64}() for _ in 1:length(params)] + S_lin_ixs = LinearIndices((n_vars, n_vars)) + A_consts = Vector{Pair{Int, T}}() + S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? - M_ind = + M_inds = any(==(Symbol("1")), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:length(params)] : nothing - - # handle constants - constants = Vector{RAMConstant}() + M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for r in partable - row_ind = col_indices[r.to] - col_ind = r.from != Symbol("1") ? col_indices[r.from] : nothing + row_ind = vars_index[r.to] + col_ind = r.from != Symbol("1") ? vars_index[r.from] : nothing if !r.free if (r.relation == :→) && (r.from == Symbol("1")) - push!(constants, RAMConstant(:M, row_ind, r.value_fixed)) + push!(M_consts, row_ind => r.value_fixed) elseif r.relation == :→ push!( - constants, - RAMConstant(:A, CartesianIndex(row_ind, col_ind), r.value_fixed), + A_consts, + A_lin_ixs[CartesianIndex(row_ind, col_ind)] => r.value_fixed, ) elseif r.relation == :↔ push!( - constants, - RAMConstant(:S, CartesianIndex(row_ind, col_ind), r.value_fixed), + S_consts, + S_lin_ixs[CartesianIndex(row_ind, col_ind)] => r.value_fixed, ) + if row_ind != col_ind # symmetric + push!( + S_consts, + S_lin_ixs[CartesianIndex(col_ind, row_ind)] => r.value_fixed, + ) + end else - error("Unsupported parameter type: $(r.relation)") + error("Unsupported relation: $(r.relation)") end else par_ind = params_index[r.param] if (r.relation == :→) && (r.from == Symbol("1")) - push!(M_ind[par_ind], row_ind) + push!(M_inds[par_ind], row_ind) elseif r.relation == :→ - push!(A_ind[par_ind], row_ind + (col_ind - 1) * n_node) + push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) elseif r.relation == :↔ - push!(S_ind[par_ind], row_ind + (col_ind - 1) * n_node) - if row_ind != col_ind - push!(S_ind[par_ind], col_ind + (row_ind - 1) * n_node) + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(row_ind, col_ind)]) + if row_ind != col_ind # symmetric + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(col_ind, row_ind)]) end else - error("Unsupported parameter type: $(r.relation)") + error("Unsupported relation: $(r.relation)") end end end + # sort linear indices + for A_ind in A_inds + sort!(A_ind) + end + for S_ind in S_inds + unique!(sort!(S_ind)) # also symmetric duplicates + end + if !isnothing(M_inds) + for M_ind in M_inds + sort!(M_ind) + end + end + sort!(A_consts, by = first) + sort!(S_consts, by = first) + if !isnothing(M_consts) + sort!(M_consts, by = first) + end - return RAMMatrices( - A_ind, - S_ind, - F_ind, - M_ind, - params, - colnames, - constants, - (n_observed, n_node), - ) + return RAMMatrices(ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), + ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), + sparse(1:n_observed, + [vars_index[var] for var in partable.observed_vars], + ones(T, n_observed), n_observed, n_vars), + !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, + params, vars_sorted) end Base.convert( @@ -255,21 +237,20 @@ Base.convert( ############################################################################################ function ParameterTable( - ram_matrices::RAMMatrices; + ram::RAMMatrices; params::Union{AbstractVector{Symbol}, Nothing} = nothing, observed_var_prefix::Symbol = :obs, latent_var_prefix::Symbol = :var, ) # defer parameter checks until we know which ones are used - if !isnothing(ram_matrices.colnames) - colnames = ram_matrices.colnames - observed_vars = colnames[ram_matrices.F_ind] - latent_vars = colnames[setdiff(eachindex(colnames), ram_matrices.F_ind)] + + if !isnothing(ram.colnames) + latent_vars = SEM.latent_vars(ram) + observed_vars = SEM.observed_vars(ram) + colnames = ram.colnames else - observed_vars = - [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram_matrices)] - latent_vars = - [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram_matrices)] + observed_vars = [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram)] + latent_vars = [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram)] colnames = vcat(observed_vars, latent_vars) end @@ -277,27 +258,16 @@ function ParameterTable( partable = ParameterTable( observed_vars = observed_vars, latent_vars = latent_vars, - params = isnothing(params) ? SEM.params(ram_matrices) : params, + params = isnothing(params) ? SEM.params(ram) : params, ) - # constants - for c in ram_matrices.constants - push!(partable, partable_row(c, colnames)) + # fill the table + append_rows!(partable, ram.S, :S, ram.params, colnames, skip_symmetric = true) + append_rows!(partable, ram.A, :A, ram.params, colnames) + if !isnothing(ram.M) + append_rows!(partable, ram.M, :M, ram.params, colnames) end - # parameters - for (i, par) in enumerate(ram_matrices.params) - append_partable_rows!( - partable, - colnames, - par, - i, - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.M_ind, - ram_matrices.size_F[2], - ) - end check_params(SEM.params(partable), partable.columns[:param]) return partable @@ -339,23 +309,13 @@ function matrix_to_relation(matrix::Symbol) end end -partable_row(c::RAMConstant, varnames::AbstractVector{Symbol}) = ( - from = varnames[c.index[2]], - relation = matrix_to_relation(c.matrix), - to = varnames[c.index[1]], - free = false, - value_fixed = c.value, - start = 0.0, - estimate = 0.0, - param = :const, -) - +# generates a ParTable row NamedTuple for a given element of RAM matrix function partable_row( - par::Symbol, - varnames::AbstractVector{Symbol}, - index::Integer, + val, + index, matrix::Symbol, - n_nod::Integer, + varnames::AbstractVector{Symbol}; + free::Bool = true, ) # variable names @@ -363,58 +323,65 @@ function partable_row( from = Symbol("1") to = varnames[index] else - cart_index = linear2cartesian(index, (n_nod, n_nod)) - - from = varnames[cart_index[2]] - to = varnames[cart_index[1]] + from = varnames[index[2]] + to = varnames[index[1]] end return ( from = from, relation = matrix_to_relation(matrix), to = to, - free = true, - value_fixed = 0.0, + free = free, + value_fixed = free ? 0.0 : val, start = 0.0, estimate = 0.0, - param = par, + param = free ? val : :const, ) end -function append_partable_rows!( +function append_rows!( partable::ParameterTable, - varnames::AbstractVector{Symbol}, - par::Symbol, - par_index::Integer, - A_ind, - S_ind, - M_ind, - n_nod::Integer, + arr::ParamsArray, + arr_name::Symbol, + params::AbstractVector, + varnames::AbstractVector{Symbol}; + skip_symmetric::Bool = false, ) - for ind in A_ind[par_index] - push!(partable, partable_row(par, varnames, ind, :A, n_nod)) - end + nparams(arr) == length(params) || throw( + ArgumentError( + "Length of parameters vector ($(length(params))) does not match the number of parameters in the matrix ($(nparams(arr)))", + ), + ) + arr_ixs = eachindex(arr) + + # add parameters + visited_indices = Set{eltype(arr_ixs)}() + for (i, par) in enumerate(params) + for j in param_occurences_range(arr, i) + arr_ix = arr_ixs[arr.linear_indices[j]] + skip_symmetric && (arr_ix ∈ visited_indices) && continue - visited_S_indices = Set{Int}() - for ind in S_ind[par_index] - if ind ∉ visited_S_indices - push!(partable, partable_row(par, varnames, ind, :S, n_nod)) - # mark index and its symmetric as visited - push!(visited_S_indices, ind) - cart_index = linear2cartesian(ind, (n_nod, n_nod)) push!( - visited_S_indices, - cartesian2linear( - CartesianIndex(cart_index[2], cart_index[1]), - (n_nod, n_nod), - ), + partable, + partable_row(par, arr_ix, arr_name, varnames, free = true), ) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) + end end end - if !isnothing(M_ind) - for ind in M_ind[par_index] - push!(partable, partable_row(par, varnames, ind, :M, n_nod)) + # add constants + for (i, val) in arr.constants + arr_ix = arr_ixs[i] + skip_symmetric && (arr_ix ∈ visited_indices) && continue + push!(partable, partable_row(val, arr_ix, arr_name, varnames, free = false)) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) end end @@ -423,14 +390,12 @@ end function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) res = ( - (mat1.A_ind == mat2.A_ind) && - (mat1.S_ind == mat2.S_ind) && - (mat1.F_ind == mat2.F_ind) && - (mat1.M_ind == mat2.M_ind) && + (mat1.A == mat2.A) && + (mat1.S == mat2.S) && + (mat1.F == mat2.F) && + (mat1.M == mat2.M) && (mat1.params == mat2.params) && - (mat1.colnames == mat2.colnames) && - (mat1.size_F == mat2.size_F) && - (mat1.constants == mat2.constants) + (mat1.colnames == mat2.colnames) ) return res end diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index e7e0b36f5..a16aac179 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -74,9 +74,6 @@ mutable struct RAM{ A5, A6, V2, - I1, - I2, - I3, M1, M2, M3, @@ -97,10 +94,6 @@ mutable struct RAM{ ram_matrices::V2 - A_indices::I1 - S_indices::I2 - M_indices::I3 - F⨉I_A⁻¹::M1 F⨉I_A⁻¹S::M2 I_A::M3 @@ -131,22 +124,14 @@ function RAM(; n_par = nparams(ram_matrices) n_obs = nobserved_vars(ram_matrices) n_var = nvars(ram_matrices) - F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 - - # get indices - A_indices = copy(ram_matrices.A_ind) - S_indices = copy(ram_matrices.S_ind) - M_indices = !isnothing(ram_matrices.M_ind) ? copy(ram_matrices.M_ind) : nothing #preallocate arrays - A_pre = zeros(n_var, n_var) - S_pre = zeros(n_var, n_var) - M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing - - set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) + nan_params = fill(NaN, n_par) + A_pre = materialize(ram_matrices.A, nan_params) + S_pre = materialize(ram_matrices.S, nan_params) + F = Matrix(ram_matrices.F) - A_pre = check_acyclic(A_pre, n_par, A_indices) + A_pre = check_acyclic(A_pre, ram_matrices.A) # pre-allocate some matrices Σ = zeros(n_obs, n_obs) @@ -155,8 +140,8 @@ function RAM(; I_A = similar(A_pre) if gradient_required - ∇A = matrix_gradient(A_indices, n_var^2) - ∇S = matrix_gradient(S_indices, n_var^2) + ∇A = sparse_gradient(ram_matrices.A) + ∇S = sparse_gradient(ram_matrices.S) else ∇A = nothing ∇S = nothing @@ -165,16 +150,16 @@ function RAM(; # μ if meanstructure MS = HasMeanStruct - !isnothing(M_indices) || throw( + !isnothing(ram_matrices.M) || throw( ArgumentError( "You set `meanstructure = true`, but your model specification contains no mean parameters.", ), ) - ∇M = gradient_required ? matrix_gradient(M_indices, n_var) : nothing + M_pre = materialize(ram_matrices.M, nan_params) + ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else MS = NoMeanStruct - M_indices = nothing M_pre = nothing μ = nothing ∇M = nothing @@ -188,9 +173,6 @@ function RAM(; μ, M_pre, ram_matrices, - A_indices, - S_indices, - M_indices, F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, @@ -206,15 +188,11 @@ end ############################################################################################ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) - fill_A_S_M!( - imply.A, - imply.S, - imply.M, - imply.A_indices, - imply.S_indices, - imply.M_indices, - params, - ) + materialize!(imply.A, imply.ram_matrices.A, params) + materialize!(imply.S, imply.ram_matrices.S, params) + if !isnothing(imply.M) + materialize!(imply.M, imply.ram_matrices.M, params) + end @. imply.I_A = -imply.A @view(imply.I_A[diagind(imply.I_A)]) .+= 1 @@ -251,12 +229,9 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre, n_par, A_indices) - # fill copy of A-matrix with random parameters - A_rand = copy(A_pre) - randpar = rand(n_par) - - fill_matrix!(A_rand, A_indices, randpar) +function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) + # fill copy of A with random parameters + A_rand = materialize(A, rand(nparams(A))) # check if the model is acyclic acyclic = isone(det(I - A_rand)) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 9a96942ae..32ffcc068 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -102,24 +102,15 @@ function RAMSymbolic(; ram_matrices = convert(RAMMatrices, specification) n_par = nparams(ram_matrices) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - par = (Symbolics.@variables θ[1:n_par])[1] - A = zeros(Num, n_var, n_var) - S = zeros(Num, n_var, n_var) - !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_var) : M = nothing - F = zeros(ram_matrices.size_F) - F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 - - set_RAMConstants!(A, S, M, ram_matrices.constants) - fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) - - A, S, F = sparse(A), sparse(S), sparse(F) + A = sparse_materialize(Num, ram_matrices.A, par) + S = sparse_materialize(Num, ram_matrices.S, par) + M = !isnothing(ram_matrices.M) ? materialize(Num, ram_matrices.M, par) : nothing + F = ram_matrices.F - if !isnothing(loss_types) - any(loss_types .<: SemWLS) ? vech = true : nothing + if !isnothing(loss_types) && any(T -> T <: SemWLS, loss_types) + vech = true end I_A⁻¹ = neumann_series(A) From 81d0ab7f89bffb3e2792573a803dbc49abfc4629 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:02:40 -0700 Subject: [PATCH 04/71] materialize!(Symm/LowTri/UpTri) --- src/additional_functions/params_array.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index f20a6518b..b79b6454f 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -135,6 +135,14 @@ materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where {T} materialize(arr::ParamsArray, param_values::AbstractVector{T}) where {T} = materialize(Union{T, eltype(arr)}, arr, param_values) +# the hack to update the structured matrix (should be fine since the structure is imposed by ParamsMatrix) +materialize!( + dest::Union{Symmetric, LowerTriangular, UpperTriangular}, + src::ParamsMatrix{<:Any}, + param_values::AbstractVector; + kwargs..., +) = materialize!(parent(dest), src, param_values; kwargs...) + function sparse_materialize( ::Type{T}, arr::ParamsMatrix, From fd13c740b6efd4c1d7406d4547ee93a0e4508acd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 16:13:34 -0700 Subject: [PATCH 05/71] ParamsArray: faster sparse materialize! --- src/additional_functions/params_array.jl | 91 ++++++++++++++++++----- src/frontend/specification/RAMMatrices.jl | 2 +- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index b79b6454f..13ae2eeaf 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -2,10 +2,14 @@ Array with partially parameterized elements. """ struct ParamsArray{T, N} <: AbstractArray{T, N} - linear_indices::Vector{Int} - param_ptr::Vector{Int} - constants::Vector{Pair{Int, T}} - size::NTuple{N, Int} + linear_indices::Vector{Int} # linear indices of the parameter refs in the destination array + nz_indices::Vector{Int} # indices of the parameters refs in nonzero elements vector + # (including the constants) ordered by the linear index + param_ptr::Vector{Int} # i-th element marks the start of the range in linear/nonzero + # indices arrays that corresponds to the i-th parameter + # (nparams + 1 elements) + constants::Vector{Tuple{Int, Int, T}} # linear index, index in nonzero vector, value + size::NTuple{N, Int} # size of the destination array end ParamsVector{T} = ParamsArray{T, 1} @@ -18,10 +22,16 @@ function ParamsArray{T, N}( ) where {T, N} params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) + param_lin_inds = reduce(vcat, params_map, init = Vector{Int}()) + nz_lin_inds = unique!(sort!([param_lin_inds; first.(constants)])) + if length(nz_lin_inds) < length(param_lin_inds) + length(constants) + throw(ArgumentError("Duplicate linear indices in the parameterized array")) + end return ParamsArray{T, N}( - reduce(vcat, params_map, init = Vector{Int}()), + param_lin_inds, + searchsortedfirst.(Ref(nz_lin_inds), param_lin_inds), params_ptr, - constants, + [(c[1], searchsortedfirst(nz_lin_inds, c[1]), c[2]) for c in constants], size, ) end @@ -58,6 +68,7 @@ ParamsArray{T}( ) where {T, N} = ParamsArray{T, N}(arr, params; kwargs...) nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 +SparseArrays.nnz(arr::ParamsArray) = length(arr.linear_indices) + length(arr.constants) Base.size(arr::ParamsArray) = arr.size Base.size(arr::ParamsArray, i::Integer) = arr.size[i] @@ -110,7 +121,7 @@ function materialize!( Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) set_zeros && fill!(dest, zero(Z)) if set_constants - @inbounds for (i, val) in src.constants + @inbounds for (i, _, val) in src.constants dest[i] = val end end @@ -122,6 +133,43 @@ function materialize!( return dest end +function materialize!( + dest::SparseMatrixCSC, + src::ParamsMatrix, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false, +) + set_zeros && throw(ArgumentError("Cannot set zeros for sparse matrix")) + size(dest) == size(src) || throw( + DimensionMismatch( + "Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match", + ), + ) + nparams(src) == length(param_values) || throw( + DimensionMismatch( + "Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))", + ), + ) + + nnz(dest) == nnz(src) || throw( + DimensionMismatch( + "Number of non-zero elements ($(nnz(dest))) does not match the number of parameter references and constants ($(nnz(src)))", + ), + ) + if set_constants + @inbounds for (_, j, val) in src.constants + dest.nzval[j] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest.nzval[src.nz_indices[j]] = val + end + end + return dest +end + """ materialize([T], src::ParamsArray{<:Any, N}, param_values::AbstractVector{T}) where T @@ -150,27 +198,30 @@ function sparse_materialize( ) where {T} nparams(arr) == length(param_values) || throw( DimensionMismatch( - "Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))", + "Number of values ($(length(param_values))) does not match the number of parameter ($(nparams(arr)))", ), ) - # constant values in sparse matrix - cvals = [T(v) for (_, v) in arr.constants] - # parameter values in sparse matrix - parvals = Vector{T}(undef, length(arr.linear_indices)) + + nz_vals = Vector{T}(undef, nnz(arr)) + nz_lininds = Vector{Int}(undef, nnz(arr)) + # fill constants + @inbounds for (lin_ind, nz_ind, val) in arr.constants + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = lin_ind + end + # fill parameters @inbounds for (i, val) in enumerate(param_values) for j in param_occurences_range(arr, i) - parvals[j] = val + nz_ind = arr.nz_indices[j] + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = arr.linear_indices[j] end end - nzixs = [first.(arr.constants); arr.linear_indices] - ixorder = sortperm(nzixs) - nzixs = nzixs[ixorder] - nzvals = [cvals; parvals][ixorder] arr_ixs = CartesianIndices(size(arr)) return sparse( - [arr_ixs[i][1] for i in nzixs], - [arr_ixs[i][2] for i in nzixs], - nzvals, + [arr_ixs[i][1] for i in nz_lininds], + [arr_ixs[i][2] for i in nz_lininds], + nz_vals, size(arr)..., ) end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index b140ae026..ee487c25e 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -374,7 +374,7 @@ function append_rows!( end # add constants - for (i, val) in arr.constants + for (i, _, val) in arr.constants arr_ix = arr_ixs[i] skip_symmetric && (arr_ix ∈ visited_indices) && continue push!(partable, partable_row(val, arr_ix, arr_name, varnames, free = false)) From 19497d5bd7fbc8450ea861a80bb0d9c8321c8d2a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Jul 2024 17:23:27 -0700 Subject: [PATCH 06/71] ParamsArray: use Iterators.flatten() (faster) --- src/additional_functions/params_array.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index 13ae2eeaf..bbee1dcf9 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -22,7 +22,7 @@ function ParamsArray{T, N}( ) where {T, N} params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init = 1), 1) - param_lin_inds = reduce(vcat, params_map, init = Vector{Int}()) + param_lin_inds = collect(Iterators.flatten(params_map)) nz_lin_inds = unique!(sort!([param_lin_inds; first.(constants)])) if length(nz_lin_inds) < length(param_lin_inds) + length(constants) throw(ArgumentError("Duplicate linear indices in the parameterized array")) From 139338d6884e45d898b6624cf43a0bbbe84d6eb7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 13:07:50 -0700 Subject: [PATCH 07/71] Base.hash(::ParamsArray) --- src/additional_functions/params_array.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index bbee1dcf9..3a58171aa 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -79,6 +79,14 @@ Base.:(==)(a::ParamsArray, b::ParamsArray) = return eltype(a) == eltype(b) && a.param_ptr == b.param_ptr && a.linear_indices == b.linear_indices +Base.hash(a::ParamsArray, h::UInt) = hash( + typeof(a), + hash( + eltype(a), + hash(size(a), hash(a.constants, hash(a.param_ptr, hash(a.linear_indices, h)))), + ), +) + # the range of arr.param_ptr indices that correspond to i-th parameter param_occurences_range(arr::ParamsArray, i::Integer) = arr.param_ptr[i]:(arr.param_ptr[i+1]-1) From 58507840210ed7d7cc31c4f9c7d9f7036d1ca8cc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 13:07:38 -0700 Subject: [PATCH 08/71] colnames -> vars --- .../start_val/start_fabin3.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 45 +++++++++---------- src/frontend/specification/documentation.jl | 4 +- test/examples/multigroup/multigroup.jl | 4 +- .../political_democracy.jl | 4 +- .../recover_parameters_twofact.jl | 2 +- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 9d692437e..d86b992da 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -129,7 +129,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) # is there at least one reference indicator? if length(reference) > 0 if (length(reference) > 1) && any(((obs, param),) -> param > 0, indicators) # don't warn if entire column is fixed - @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" + @warn "You have more than 1 scaling indicator for $(ram_matrices.vars[i])" end ref = reference[1] diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index ee487c25e..451f5fd69 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -9,7 +9,7 @@ struct RAMMatrices <: SemSpecification F::SparseMatrixCSC{Float64} M::Union{ParamsVector{Float64}, Nothing} params::Vector{Symbol} - colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) + vars::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_vars()) end nparams(ram::RAMMatrices) = nparams(ram.A) @@ -17,7 +17,7 @@ nvars(ram::RAMMatrices) = size(ram.F, 2) nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) -vars(ram::RAMMatrices) = ram.colnames +vars(ram::RAMMatrices) = ram.vars isobserved_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] > ram.F.colptr[i] islatent_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] == ram.F.colptr[i] @@ -34,13 +34,12 @@ function observed_var_indices(ram::RAMMatrices) return obs_inds end -latent_var_indices(ram::RAMMatrices) = - [i for i in axes(ram.F, 2) if islatent_var(ram, i)] +latent_var_indices(ram::RAMMatrices) = [i for i in axes(ram.F, 2) if islatent_var(ram, i)] # observed variables in the order as they appear in ram.F rows function observed_vars(ram::RAMMatrices) - if isnothing(ram.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + if isnothing(ram.vars) + @warn "Your RAMMatrices do not contain variable names. Please make sure the order of variables in your data is correct!" return nothing else obs_vars = Vector{Symbol}(undef, nobserved_vars(ram)) @@ -55,11 +54,11 @@ function observed_vars(ram::RAMMatrices) end function latent_vars(ram::RAMMatrices) - if isnothing(ram.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + if isnothing(ram.vars) + @warn "Your RAMMatrices do not contain variable names. Please make sure the order of variables in your data is correct!" return nothing else - return [col for (i, col) in enumerate(ram.colnames) if islatent_var(ram, i)] + return [col for (i, col) in enumerate(ram.vars) if islatent_var(ram, i)] end end @@ -73,32 +72,32 @@ function RAMMatrices(; F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, params::AbstractVector{Symbol}, - colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + vars::Union{AbstractVector{Symbol}, Nothing} = nothing, ) ncols = size(A, 2) - isnothing(colnames) || check_vars(colnames, ncols) + isnothing(vars) || check_vars(vars, ncols) size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) size(A, 2) == ncols || throw( DimensionMismatch( - "A should have as many rows and columns as colnames length ($ncols), $(size(A)) found", + "A should have as many rows and columns as vars length ($ncols), $(size(A)) found", ), ) size(S, 2) == ncols || throw( DimensionMismatch( - "S should have as many rows and columns as colnames length ($ncols), $(size(S)) found", + "S should have as many rows and columns as vars length ($ncols), $(size(S)) found", ), ) size(F, 2) == ncols || throw( DimensionMismatch( - "F should have as many columns as colnames length ($ncols), $(size(F, 2)) found", + "F should have as many columns as vars length ($ncols), $(size(F, 2)) found", ), ) if !isnothing(M) length(M) == ncols || throw( DimensionMismatch( - "M should have as many elements as colnames length ($ncols), $(length(M)) found", + "M should have as many elements as vars length ($ncols), $(length(M)) found", ), ) end @@ -111,7 +110,7 @@ function RAMMatrices(; if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, params, colnames) + return RAMMatrices(A, S, F, M, params, vars) end ############################################################################################ @@ -244,14 +243,14 @@ function ParameterTable( ) # defer parameter checks until we know which ones are used - if !isnothing(ram.colnames) + if !isnothing(ram.vars) latent_vars = SEM.latent_vars(ram) observed_vars = SEM.observed_vars(ram) - colnames = ram.colnames + vars = ram.vars else observed_vars = [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram)] latent_vars = [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram)] - colnames = vcat(observed_vars, latent_vars) + vars = vcat(observed_vars, latent_vars) end # construct an empty table @@ -262,10 +261,10 @@ function ParameterTable( ) # fill the table - append_rows!(partable, ram.S, :S, ram.params, colnames, skip_symmetric = true) - append_rows!(partable, ram.A, :A, ram.params, colnames) + append_rows!(partable, ram.S, :S, ram.params, vars, skip_symmetric = true) + append_rows!(partable, ram.A, :A, ram.params, vars) if !isnothing(ram.M) - append_rows!(partable, ram.M, :M, ram.params, colnames) + append_rows!(partable, ram.M, :M, ram.params, vars) end check_params(SEM.params(partable), partable.columns[:param]) @@ -395,7 +394,7 @@ function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) (mat1.F == mat2.F) && (mat1.M == mat2.M) && (mat1.params == mat2.params) && - (mat1.colnames == mat2.colnames) + (mat1.vars == mat2.vars) ) return res end diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index e869dd43f..46135ead0 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -95,7 +95,7 @@ function EnsembleParameterTable end (1) RAMMatrices(partable::ParameterTable) - (2) RAMMatrices(;A, S, F, M = nothing, params, colnames) + (2) RAMMatrices(;A, S, F, M = nothing, params, vars) (3) RAMMatrices(partable::EnsembleParameterTable) @@ -110,7 +110,7 @@ Return `RAMMatrices` constructed from (1) a parameter table or (2) individual ma - `F`: filter matrix - `M`: vector of mean effects - `params::Vector{Symbol}`: parameter labels -- `colnames::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns +- `vars::Vector{Symbol}`: variable names corresponding to the A, S and F matrix columns # Examples See the online documentation on [Model specification](@ref) and the [RAMMatrices interface](@ref). diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index a2f277d91..caaa5c3f7 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -60,7 +60,7 @@ specification_g1 = RAMMatrices(; S = S1, F = F, params = x, - colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], + vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) specification_g2 = RAMMatrices(; @@ -68,7 +68,7 @@ specification_g2 = RAMMatrices(; S = S2, F = F, params = x, - colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], + vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) partable = EnsembleParameterTable( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index d7fbb8f2c..2f570302a 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -76,7 +76,7 @@ spec = RAMMatrices(; S = S, F = F, params = x, - colnames = [ + vars = [ :x1, :x2, :x3, @@ -108,7 +108,7 @@ spec_mean = RAMMatrices(; F = F, M = M, params = x, - colnames = [ + vars = [ :x1, :x2, :x3, diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index f00187fac..89c1225e2 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -40,7 +40,7 @@ A = [ 0 0 0 0 0 0 0 0 ] -ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, colnames = nothing) +ram_matrices = RAMMatrices(; A = A, S = S, F = F, params = x, vars = nothing) true_val = [ repeat([1], 8) From 0f747b7ec789d5aceb4eed3b2b4612de2f7c60f8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:50:18 -0700 Subject: [PATCH 09/71] update_partable!(): better params unique check --- src/frontend/specification/ParameterTable.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index df2cc165b..05350fb12 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -309,10 +309,10 @@ function update_partable!( "The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same", ), ) + dup_params = nonunique(params) + isempty(dup_params) || + throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) param_values = Dict(zip(params, values)) - if length(param_values) != length(params) - throw(ArgumentError("Duplicate parameter names in `params`")) - end update_partable!(partable, column, param_values, default) end From aa34d5352979399de3f40226bf05b5690b777b9f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 31 Jul 2024 21:35:50 -0700 Subject: [PATCH 10/71] start_fabin3: check obs_mean data & meanstructure --- src/additional_functions/start_val/start_fabin3.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index d86b992da..53cf7cff6 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -30,13 +30,21 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k return start_fabin3(imply.ram_matrices, observed.em_model.Σ, observed.em_model.μ) end -function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) +function start_fabin3( + ram_matrices::RAMMatrices, + Σ::AbstractMatrix, + μ::Union{AbstractVector, Nothing}, +) A, S, F, M, n_par = ram_matrices.A, ram_matrices.S, ram_matrices.F, ram_matrices.M, nparams(ram_matrices) + if !isnothing(M) && isnothing(μ) + throw(ArgumentError("RAM has meanstructure, but no observed means provided.")) + end + start_val = zeros(n_par) F_var2obs = Dict( i => F.rowval[F.colptr[i]] for i in axes(F, 2) if isobserved_var(ram_matrices, i) From 13aacd0cf87c1f8946f97bc27eb2cfe57f2e21cd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Nov 2024 11:28:11 -0800 Subject: [PATCH 11/71] params/vars API tweaks and tests --- .../start_val/start_partable.jl | 31 +++++++------------ src/frontend/specification/ParameterTable.jl | 4 +-- src/frontend/specification/RAMMatrices.jl | 2 +- src/frontend/specification/StenoGraphs.jl | 2 +- test/examples/helper.jl | 2 ++ test/examples/multigroup/build_models.jl | 2 ++ .../political_democracy/constructor.jl | 1 + .../political_democracy.jl | 12 +++++-- 8 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 6fb15e365..15f863f5b 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -21,28 +21,19 @@ function start_parameter_table(observed, imply, optimizer, args...; kwargs...) return start_parameter_table(ram_matrices(imply); kwargs...) end -function start_parameter_table( - ram_matrices::RAMMatrices; - parameter_table::ParameterTable, - kwargs..., -) +function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) start_val = zeros(0) - for param in ram_matrices.params - found = false - for (i, param_table) in enumerate(parameter_table.params) - if param == param_table - push!(start_val, parameter_table.start[i]) - found = true - break - end - end - if !found - throw( - ErrorException( - "At least one parameter could not be found in the parameter table.", - ), - ) + param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) + start_col = partable.columns[:start] + + for (i, param) in enumerate(partable.columns[:param]) + par_ind = get(param_indices, param, nothing) + if !isnothing(par_ind) + par_start = start_col[i] + isfinite(par_start) && (start_val[i] = par_start) + else + throw(ErrorException("Parameter $(param) is not in the parameter table.")) end end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 05350fb12..8b7cc0973 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -54,8 +54,8 @@ function ParameterTable( return ParameterTable( Dict(col => copy(values) for (col, values) in pairs(partable.columns)), - observed_vars = copy(partable.observed_vars), - latent_vars = copy(partable.latent_vars), + observed_vars = copy(observed_vars(partable)), + latent_vars = copy(latent_vars(partable)), params = params, ) end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 451f5fd69..0c5722f57 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -110,7 +110,7 @@ function RAMMatrices(; if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, params, vars) + return RAMMatrices(A, S, F, M, copy(params), vars) end ############################################################################################ diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 64a33f13e..5cf87c07a 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -42,7 +42,7 @@ function ParameterTable( latent_vars::AbstractVector{Symbol}, params::Union{AbstractVector{Symbol}, Nothing} = nothing, group::Union{Integer, Nothing} = nothing, - param_prefix = :θ, + param_prefix::Symbol = :θ, ) graph = unique(graph) n = length(graph) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index d4c140d67..042f7005f 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,6 +1,8 @@ using LinearAlgebra: norm function test_gradient(model, params; rtol = 1e-10, atol = 0) + @test nparams(model) == length(params) + true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), params) gradient = similar(params) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 3f29a6898..6991dd479 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -8,6 +8,8 @@ model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbo model_g2 = Sem(specification = specification_g2, data = dat_g2, imply = RAM) +@test SEM.params(model_g1.imply.ram_matrices) == SEM.params(model_g2.imply.ram_matrices) + # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) model_ml_multigroup2 = SemEnsemble( diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 99ef06b3a..bebabf6e0 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -6,6 +6,7 @@ using Random ############################################################################################ model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) +@test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( specification = spec, diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 2f570302a..2265e2a59 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Test, FiniteDiff +SEM = StructuralEquationModels + include( joinpath( chop(dirname(pathof(StructuralEquationModels)), tail = 3), @@ -96,9 +98,9 @@ spec = RAMMatrices(; partable = ParameterTable(spec) -# w. meanstructure ------------------------------------------------------------------------- +@test SEM.params(spec) == SEM.params(partable) -x = Symbol.("x" .* string.(1:38)) +# w. meanstructure ------------------------------------------------------------------------- M = [:x32; :x33; :x34; :x35; :x36; :x37; :x38; :x35; :x36; :x37; :x38; 0.0; 0.0; 0.0] @@ -107,7 +109,7 @@ spec_mean = RAMMatrices(; S = S, F = F, M = M, - params = x, + params = [SEM.params(spec); Symbol.("x", string.(32:38))], vars = [ :x1, :x2, @@ -128,6 +130,8 @@ spec_mean = RAMMatrices(; partable_mean = ParameterTable(spec_mean) +@test SEM.params(partable_mean) == SEM.params(spec_mean) + start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] @@ -164,6 +168,8 @@ end spec = ParameterTable(spec) spec_mean = ParameterTable(spec_mean) +@test SEM.params(spec) == SEM.params(partable) + partable = spec partable_mean = spec_mean From 8c26c351c5f440df66bc6cb78a92b43007b66c00 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:10:43 -0700 Subject: [PATCH 12/71] generic imply: keep F sparse --- src/imply/RAM/generic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index a16aac179..850934a9c 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -129,7 +129,7 @@ function RAM(; nan_params = fill(NaN, n_par) A_pre = materialize(ram_matrices.A, nan_params) S_pre = materialize(ram_matrices.S, nan_params) - F = Matrix(ram_matrices.F) + F = copy(ram_matrices.F) A_pre = check_acyclic(A_pre, ram_matrices.A) From 3816539d9d6cacb0ee9740e6cb27108343f00b4f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 16:49:33 -0700 Subject: [PATCH 13/71] tests helper: is_extended_tests() to consolidate ENV variable check --- test/examples/helper.jl | 4 ++++ test/examples/political_democracy/political_democracy.jl | 6 +++--- test/runtests.jl | 3 --- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 042f7005f..f35d2cac6 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,5 +1,9 @@ using LinearAlgebra: norm +function is_extended_tests() + return lowercase(get(ENV, "JULIA_EXTENDED_TESTS", "false")) == "true" +end + function test_gradient(model, params; rtol = 1e-10, atol = 0) @test nparams(model) == length(params) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 2265e2a59..6754c29c3 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -146,7 +146,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") @@ -182,7 +182,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") @@ -269,7 +269,7 @@ semoptimizer = SemOptimizerNLopt include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") diff --git a/test/runtests.jl b/test/runtests.jl index c3b15475f..28d2142b1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,6 +11,3 @@ end @time @safetestset "Example Models" begin include("examples/examples.jl") end - -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" -end From b8d9a8fa9d60b8779a9a5a086f553cbb184fd08c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 25 May 2024 17:05:25 -0700 Subject: [PATCH 14/71] Optim sem_fit(): use provided optimizer --- src/optimizer/optim.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index bb1bf507e..7951e6b14 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -31,8 +31,8 @@ function sem_fit( result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_val, - model.optimizer.algorithm, - model.optimizer.options, + optim.algorithm, + optim.options, ) return SemFit(result, model, start_val) end From dd275d57dca12c473fe5d85b38db00493e15dd4e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:40:52 -0800 Subject: [PATCH 15/71] prepare_start_params(): arg-dependent dispatch * convert to argument type-dependent dispatch * replace start_val() function with prepare_start_params() * refactor start_parameter_table() into prepare_start_params(start_val::ParameterTable, ...) and use the SEM model param indices * unify processing of starting values by all optimizers * support dictionaries of values --- src/StructuralEquationModels.jl | 3 - .../start_val/start_partable.jl | 41 ------------- .../start_val/start_val.jl | 26 -------- src/optimizer/NLopt.jl | 17 ++---- src/optimizer/documentation.jl | 60 +++++++++++++++++-- src/optimizer/optim.jl | 12 ++-- 6 files changed, 65 insertions(+), 94 deletions(-) delete mode 100644 src/additional_functions/start_val/start_partable.jl delete mode 100644 src/additional_functions/start_val/start_val.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 6172af1ea..3f68dd95f 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -70,9 +70,7 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") -include("additional_functions/start_val/start_partable.jl") include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") include("additional_functions/simulation.jl") @@ -109,7 +107,6 @@ export AbstractSem, start_val, start_fabin3, start_simple, - start_parameter_table, SemLoss, SemLossFunction, SemML, diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl deleted file mode 100644 index 15f863f5b..000000000 --- a/src/additional_functions/start_val/start_partable.jl +++ /dev/null @@ -1,41 +0,0 @@ -""" - start_parameter_table(model; parameter_table) - -Return a vector of starting values taken from `parameter_table`. -""" -function start_parameter_table end - -# splice model and loss functions -function start_parameter_table(model::AbstractSemSingle; kwargs...) - return start_parameter_table( - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., - ) -end - -# RAM(Symbolic) -function start_parameter_table(observed, imply, optimizer, args...; kwargs...) - return start_parameter_table(ram_matrices(imply); kwargs...) -end - -function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) - start_val = zeros(0) - - param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) - start_col = partable.columns[:start] - - for (i, param) in enumerate(partable.columns[:param]) - par_ind = get(param_indices, param, nothing) - if !isnothing(par_ind) - par_start = start_col[i] - isfinite(par_start) && (start_val[i] = par_start) - else - throw(ErrorException("Parameter $(param) is not in the parameter table.")) - end - end - - return start_val -end diff --git a/src/additional_functions/start_val/start_val.jl b/src/additional_functions/start_val/start_val.jl deleted file mode 100644 index 8b6402efa..000000000 --- a/src/additional_functions/start_val/start_val.jl +++ /dev/null @@ -1,26 +0,0 @@ -""" - start_val(model) - -Return a vector of starting values. -Defaults are FABIN 3 starting values for single models and simple starting values for -ensemble models. -""" -function start_val end -# Single Models ---------------------------------------------------------------------------- - -# splice model and loss functions -start_val(model::AbstractSemSingle; kwargs...) = start_val( - model, - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., -) - -# Fabin 3 starting values for RAM(Symbolic) -start_val(model, observed, imply, optimizer, args...; kwargs...) = - start_fabin3(model; kwargs...) - -# Ensemble Models -------------------------------------------------------------------------- -start_val(model::SemEnsemble; kwargs...) = start_simple(model; kwargs...) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index 7f4f61e1e..6b03a676c 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -25,21 +25,16 @@ end # sem_fit method function sem_fit( optimizer::SemOptimizerNLopt, - model::AbstractSem; - start_val = start_val, + model::AbstractSem, + start_params::AbstractVector; kwargs..., ) - # starting values - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end - # construct the NLopt problem opt = construct_NLopt_problem( model.optimizer.algorithm, model.optimizer.options, - length(start_val), + length(start_params), ) set_NLopt_constraints!(opt, model.optimizer) opt.min_objective = @@ -55,15 +50,15 @@ function sem_fit( opt_local = construct_NLopt_problem( model.optimizer.local_algorithm, model.optimizer.local_options, - length(start_val), + length(start_params), ) opt.local_optimizer = opt_local end # fit - result = NLopt.optimize(opt, start_val) + result = NLopt.optimize(opt, start_params) - return SemFit_NLopt(result, model, start_val, opt) + return SemFit_NLopt(result, model, start_params, opt) end ############################################################################################ diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index 7c17e6ce2..a369fba77 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -5,16 +5,17 @@ Return the fitted `model`. # Arguments - `model`: `AbstractSem` to fit -- `start_val`: vector of starting values or function to compute starting values (1) +- `start_val`: a vector or a dictionary of starting parameter values, + or function to compute them (1) - `kwargs...`: keyword arguments, passed to starting value functions -(1) available options are `start_fabin3`, `start_simple` and `start_partable`. +(1) available functions are `start_fabin3`, `start_simple` and `start_partable`. For more information, we refer to the individual documentations and the online documentation on [Starting values](@ref). # Examples ```julia sem_fit( - my_model; + my_model; start_val = start_simple, start_covariances_latent = 0.5) ``` @@ -22,8 +23,57 @@ sem_fit( function sem_fit end # dispatch on optimizer -sem_fit(model::AbstractSem; kwargs...) = sem_fit(model.optimizer, model; kwargs...) +function sem_fit(model::AbstractSem; start_val = nothing, kwargs...) + start_params = prepare_start_params(start_val, model; kwargs...) + @assert start_params isa AbstractVector + @assert length(start_params) == nparams(model) + + sem_fit(model.optimizer, model, start_params; kwargs...) +end # fallback method -sem_fit(optimizer::SemOptimizer, model::AbstractSem; kwargs...) = +sem_fit(optimizer::SemOptimizer, model::AbstractSem, start_params; kwargs...) = error("Optimizer $(optimizer) support not implemented.") + +# FABIN3 is the default method for single models +prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = + start_fabin3(model; kwargs...) + +# simple algorithm is the default method for ensembles +prepare_start_params(start_val::Nothing, model::AbstractSem; kwargs...) = + start_simple(model; kwargs...) + +function prepare_start_params(start_val::AbstractVector, model::AbstractSem; kwargs...) + (length(start_val) == nparams(model)) || throw( + DimensionMismatch( + "The length of `start_val` vector ($(length(start_val))) does not match the number of model parameters ($(nparams(model))).", + ), + ) + return start_val +end + +function prepare_start_params(start_val::AbstractDict, model::AbstractSem; kwargs...) + return [start_val[param] for param in params(model)] # convert to a vector +end + +# get from the ParameterTable (potentially from a different model with match param names) +# TODO: define kwargs that instruct to get values from "estimate" and "fixed" +function prepare_start_params(start_val::ParameterTable, model::AbstractSem; kwargs...) + res = zeros(eltype(start_val.columns[:start]), nparams(model)) + param_indices = Dict(param => i for (i, param) in enumerate(params(model))) + + for (param, startval) in zip(start_val.columns[:param], start_val.columns[:start]) + (param == :const) && continue + par_ind = get(param_indices, param, nothing) + if !isnothing(par_ind) + isfinite(startval) && (res[par_ind] = startval) + else + throw( + ErrorException( + "Model parameter $(param) not found in the parameter table.", + ), + ) + end + end + return res +end diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 7951e6b14..b2adfe03a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -20,19 +20,15 @@ convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) function sem_fit( optim::SemOptimizerOptim, - model::AbstractSem; - start_val = start_val, + model::AbstractSem, + start_params::AbstractVector; kwargs..., ) - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end - result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), - start_val, + start_params, optim.algorithm, optim.options, ) - return SemFit(result, model, start_val) + return SemFit(result, model, start_params) end From 0131bb784d49286b527f06f251f59f9b9b11cd55 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:45:37 -0800 Subject: [PATCH 16/71] prepare_param_bounds() API for optim --- src/optimizer/documentation.jl | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index a369fba77..cf6aaa312 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -77,3 +77,40 @@ function prepare_start_params(start_val::ParameterTable, model::AbstractSem; kwa end return res end + +# prepare a vector of model parameter bounds (BOUND=:lower or BOUND=:lower): +# use the user-specified "bounds" vector "as is" +function prepare_param_bounds( + ::Val{BOUND}, + bounds::AbstractVector{<:Number}, + model::AbstractSem; + default::Number, # unused for vector bounds + variance_default::Number, # unused for vector bounds +) where {BOUND} + length(bounds) == nparams(model) || throw( + DimensionMismatch( + "The length of `bounds` vector ($(length(bounds))) does not match the number of model parameters ($(nparams(model))).", + ), + ) + return bounds +end + +# prepare a vector of model parameter bounds +# given the "bounds" dictionary and default values +function prepare_param_bounds( + ::Val{BOUND}, + bounds::Union{AbstractDict, Nothing}, + model::AbstractSem; + default::Number, + variance_default::Number, +) where {BOUND} + varparams = Set(variance_params(model.imply.ram_matrices)) + res = [ + begin + def = in(p, varparams) ? variance_default : default + isnothing(bounds) ? def : get(bounds, p, def) + end for p in SEM.params(model) + ] + + return res +end From fbdcc7f9b8caacfe8b4d553e3d323bbf848852f9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:45:37 -0800 Subject: [PATCH 17/71] u/l_bounds support for Optim.jl --- src/optimizer/optim.jl | 45 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index b2adfe03a..19623b965 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -22,13 +22,46 @@ function sem_fit( optim::SemOptimizerOptim, model::AbstractSem, start_params::AbstractVector; + lower_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + upper_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + lower_bound = -Inf, + upper_bound = Inf, + variance_lower_bound::Number = 0.0, + variance_upper_bound::Number = Inf, kwargs..., ) - result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), - start_params, - optim.algorithm, - optim.options, - ) + # setup lower/upper bounds if the algorithm supports it + if optim.algorithm isa Optim.Fminbox || optim.algorithm isa Optim.SAMIN + lbounds = prepare_param_bounds( + Val(:lower), + lower_bounds, + model, + default = lower_bound, + variance_default = variance_lower_bound, + ) + ubounds = prepare_param_bounds( + Val(:upper), + upper_bounds, + model, + default = upper_bound, + variance_default = variance_upper_bound, + ) + start_params = clamp.(start_params, lbounds, ubounds) + result = Optim.optimize( + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + lbounds, + ubounds, + start_params, + optim.algorithm, + optim.options, + ) + else + result = Optim.optimize( + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + start_params, + optim.algorithm, + optim.options, + ) + end return SemFit(result, model, start_params) end From d1f323a7b8f3f35d813ec130056509b16975c9d9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 15:52:01 -0700 Subject: [PATCH 18/71] SemOptimizer(engine = ...) ctor --- src/diff/Empty.jl | 4 ++-- src/diff/NLopt.jl | 4 +++- src/diff/optim.jl | 4 +++- src/types.jl | 13 +++++++++- test/Project.toml | 1 + test/examples/political_democracy/by_parts.jl | 9 +++---- .../political_democracy/constraints.jl | 11 +++++---- .../political_democracy/constructor.jl | 6 +++-- .../political_democracy.jl | 24 +++++++++---------- 9 files changed, 49 insertions(+), 27 deletions(-) diff --git a/src/diff/Empty.jl b/src/diff/Empty.jl index 57fa9ee98..45a20db55 100644 --- a/src/diff/Empty.jl +++ b/src/diff/Empty.jl @@ -15,13 +15,13 @@ an optimizer part. Subtype of `SemOptimizer`. """ -struct SemOptimizerEmpty <: SemOptimizer end +struct SemOptimizerEmpty <: SemOptimizer{:Empty} end ############################################################################################ ### Constructor ############################################################################################ -# SemOptimizerEmpty(;kwargs...) = SemOptimizerEmpty() +SemOptimizer{:Empty}() = SemOptimizerEmpty() ############################################################################################ ### Recommended methods diff --git a/src/diff/NLopt.jl b/src/diff/NLopt.jl index 12fcd7e0f..f0e4cea5b 100644 --- a/src/diff/NLopt.jl +++ b/src/diff/NLopt.jl @@ -56,7 +56,7 @@ see [Constrained optimization](@ref) in our online documentation. Subtype of `SemOptimizer`. """ -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} algorithm::A local_algorithm::A2 options::B @@ -97,6 +97,8 @@ function SemOptimizerNLopt(; ) end +SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) + ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/diff/optim.jl b/src/diff/optim.jl index 4e4b04e9f..5b8845275 100644 --- a/src/diff/optim.jl +++ b/src/diff/optim.jl @@ -44,11 +44,13 @@ my_newton_optimizer = SemOptimizerOptim( Subtype of `SemOptimizer`. """ -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} algorithm::A options::B end +SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) + SemOptimizerOptim(; algorithm = LBFGS(), options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), diff --git a/src/types.jl b/src/types.jl index 576252726..90b648ac8 100644 --- a/src/types.jl +++ b/src/types.jl @@ -84,7 +84,18 @@ Supertype of all objects that can serve as the `optimizer` field of a SEM. Connects the SEM to its optimization backend and controls options like the optimization algorithm. If you want to connect the SEM package to a new optimization backend, you should implement a subtype of SemOptimizer. """ -abstract type SemOptimizer end +abstract type SemOptimizer{E} end + +engine(::Type{SemOptimizer{E}}) where {E} = E +engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) + +SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = + SemOptimizer{engine}(args...; kwargs...) + +# fallback optimizer constructor +function SemOptimizer{E}(args...; kwargs...) where {E} + throw(ErrorException("$E optimizer is not supported.")) +end """ Supertype of all objects that can serve as the observed field of a SEM. diff --git a/test/Project.toml b/test/Project.toml index c5124c659..5867c1f40 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -6,6 +6,7 @@ JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index f50fb6dd0..87e5fb733 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -25,7 +25,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- @@ -152,10 +152,11 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches - optimizer_obj = SemOptimizerOptim( + optimizer_obj = SemOptimizer( + engine = opt_engine, algorithm = Newton(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang(), @@ -220,7 +221,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index e5cd96ab9..47f27582a 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -1,4 +1,5 @@ # NLopt constraints ------------------------------------------------------------------------ +using NLopt # 1.5*x1 == x2 (aka 1.5*x1 - x2 == 0) #= function eq_constraint(x, grad) @@ -20,12 +21,13 @@ function ineq_constraint(x, grad) 0.6 - x[30] * x[31] end -constrained_optimizer = SemOptimizerNLopt(; +constrained_optimizer = SemOptimizer(; + engine = :NLopt, algorithm = :AUGLAG, local_algorithm = :LD_LBFGS, options = Dict(:xtol_rel => 1e-4), - # equality_constraints = NLoptConstraint(;f = eq_constraint, tol = 1e-14), - inequality_constraints = NLoptConstraint(; f = ineq_constraint, tol = 1e-8), + # equality_constraints = (f = eq_constraint, tol = 1e-14), + inequality_constraints = (f = ineq_constraint, tol = 0.0), ) model_ml_constrained = @@ -38,7 +40,8 @@ solution_constrained = sem_fit(model_ml_constrained) model_ml_maxeval = Sem( specification = spec, data = dat, - optimizer = SemOptimizerNLopt, + optimizer = SemOptimizer, + engine = :NLopt, options = Dict(:maxeval => 10), ) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index bebabf6e0..5ed576dc1 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,10 +1,12 @@ using Statistics: cov, mean -using Random +using Random, NLopt ############################################################################################ ### models w.o. meanstructure ############################################################################################ +semoptimizer = SemOptimizer(engine = opt_engine) + model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) @test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) @@ -205,7 +207,7 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches model_ls = Sem( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 6754c29c3..a2e5089bb 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -136,22 +136,22 @@ start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0. start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices | parts | NLopt" begin include("by_parts.jl") end @@ -173,21 +173,21 @@ spec_mean = ParameterTable(spec_mean) partable = spec partable_mean = spec_mean -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | parts | NLopt" begin include("by_parts.jl") end @@ -260,21 +260,21 @@ start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)] start_test_mean = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 3); fill(0.05, 13)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "Graph → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "Graph → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "Graph → ParameterTable | parts | NLopt" begin include("by_parts.jl") end From 0a6b073d2ac2ef4d361faf0f7d1d623a5889d72d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Apr 2024 18:33:50 -0700 Subject: [PATCH 19/71] SEMNLOptExt for NLopt --- Project.toml | 7 ++++++- ext/SEMNLOptExt.jl | 12 +++++++++++ {src => ext}/diff/NLopt.jl | 37 ++++++++++++++++++++------------- {src => ext}/optimizer/NLopt.jl | 20 +++++++++--------- src/StructuralEquationModels.jl | 5 ----- 5 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 ext/SEMNLOptExt.jl rename {src => ext}/diff/NLopt.jl (76%) rename {src => ext}/optimizer/NLopt.jl (84%) diff --git a/Project.toml b/Project.toml index b038c3364..21bd43814 100644 --- a/Project.toml +++ b/Project.toml @@ -12,7 +12,6 @@ LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c" -NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" @@ -44,3 +43,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test"] + +[weakdeps] +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" + +[extensions] +SEMNLOptExt = "NLopt" diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt.jl new file mode 100644 index 000000000..dfc3bbb42 --- /dev/null +++ b/ext/SEMNLOptExt.jl @@ -0,0 +1,12 @@ +module SEMNLOptExt + +using StructuralEquationModels, NLopt + +SEM = StructuralEquationModels + +export SemOptimizerNLopt, NLoptConstraint + +include("diff/NLopt.jl") +include("optimizer/NLopt.jl") + +end diff --git a/src/diff/NLopt.jl b/ext/diff/NLopt.jl similarity index 76% rename from src/diff/NLopt.jl rename to ext/diff/NLopt.jl index f0e4cea5b..8267cf4bc 100644 --- a/src/diff/NLopt.jl +++ b/ext/diff/NLopt.jl @@ -9,10 +9,10 @@ Connects to `NLopt.jl` as the optimization backend. SemOptimizerNLopt(; algorithm = :LD_LBFGS, options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), kwargs...) # Arguments @@ -37,9 +37,9 @@ my_constrained_optimizer = SemOptimizerNLopt(; ``` # Usage -All algorithms and options from the NLopt library are available, for more information see +All algorithms and options from the NLopt library are available, for more information see the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, +For information on how to use inequality and equality constraints, see [Constrained optimization](@ref) in our online documentation. # Extended help @@ -65,11 +65,16 @@ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} inequality_constraints::C end -Base.@kwdef mutable struct NLoptConstraint +Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 end +Base.convert( + ::Type{NLoptConstraint}, + tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, +) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) + ############################################################################################ ### Constructor ############################################################################################ @@ -83,35 +88,37 @@ function SemOptimizerNLopt(; inequality_constraints = Vector{NLoptConstraint}(), kwargs..., ) - applicable(iterate, equality_constraints) || + applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) || + applicable(iterate, inequality_constraints) && + !isa(inequality_constraints, NamedTuple) || (inequality_constraints = [inequality_constraints]) return SemOptimizerNLopt( algorithm, local_algorithm, options, local_options, - equality_constraints, - inequality_constraints, + convert.(NLoptConstraint, equality_constraints), + convert.(NLoptConstraint, inequality_constraints), ) end -SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) +SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = optimizer +SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = + optimizer ############################################################################################ ### additional methods ############################################################################################ -algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm +SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm -options(optimizer::SemOptimizerNLopt) = optimizer.options +SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints diff --git a/src/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl similarity index 84% rename from src/optimizer/NLopt.jl rename to ext/optimizer/NLopt.jl index 6b03a676c..1abdac053 100644 --- a/src/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -7,9 +7,9 @@ mutable struct NLoptResult problem::Any end -optimizer(res::NLoptResult) = res.problem.algorithm -n_iterations(res::NLoptResult) = res.problem.numevals -convergence(res::NLoptResult) = res.result[3] +SEM.optimizer(res::NLoptResult) = res.problem.algorithm +SEM.n_iterations(res::NLoptResult) = res.problem.numevals +SEM.convergence(res::NLoptResult) = res.result[3] # construct SemFit from fitted NLopt object function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) @@ -23,7 +23,7 @@ function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) end # sem_fit method -function sem_fit( +function SEM.sem_fit( optimizer::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; @@ -38,7 +38,7 @@ function sem_fit( ) set_NLopt_constraints!(opt, model.optimizer) opt.min_objective = - (par, G) -> evaluate!( + (par, G) -> SEM.evaluate!( eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, @@ -68,19 +68,19 @@ end function construct_NLopt_problem(algorithm, options, npar) opt = Opt(algorithm, npar) - for key in keys(options) - setproperty!(opt, key, options[key]) + for (key, val) in pairs(options) + setproperty!(opt, key, val) end return opt end -function set_NLopt_constraints!(opt, optimizer::SemOptimizerNLopt) +function set_NLopt_constraints!(opt::Opt, optimizer::SemOptimizerNLopt) for con in optimizer.inequality_constraints - inequality_constraint!(opt::Opt, con.f, con.tol) + inequality_constraint!(opt, con.f, con.tol) end for con in optimizer.equality_constraints - equality_constraint!(opt::Opt, con.f, con.tol) + equality_constraint!(opt, con.f, con.tol) end end diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 3f68dd95f..ec2abf31c 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -7,7 +7,6 @@ using LinearAlgebra, StatsBase, SparseArrays, Symbolics, - NLopt, FiniteDiff, PrettyTables, Distributions, @@ -62,12 +61,10 @@ include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer include("diff/optim.jl") -include("diff/NLopt.jl") include("diff/Empty.jl") # optimizer include("optimizer/documentation.jl") include("optimizer/optim.jl") -include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") include("additional_functions/start_val/start_fabin3.jl") @@ -119,8 +116,6 @@ export AbstractSem, SemOptimizer, SemOptimizerEmpty, SemOptimizerOptim, - SemOptimizerNLopt, - NLoptConstraint, optimizer, n_iterations, convergence, From 730eadccbfbd760eaf29ecff5f04fe5031e43228 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Nov 2024 18:07:38 -0800 Subject: [PATCH 20/71] NLopt: sem_fit(): use provided optimizer --- ext/optimizer/NLopt.jl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ext/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl index 1abdac053..94da98361 100644 --- a/ext/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -24,32 +24,28 @@ end # sem_fit method function SEM.sem_fit( - optimizer::SemOptimizerNLopt, + optim::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; kwargs..., ) # construct the NLopt problem - opt = construct_NLopt_problem( - model.optimizer.algorithm, - model.optimizer.options, - length(start_params), - ) - set_NLopt_constraints!(opt, model.optimizer) + opt = construct_NLopt_problem(optim.algorithm, optim.options, length(start_params)) + set_NLopt_constraints!(opt, optim) opt.min_objective = (par, G) -> SEM.evaluate!( - eltype(par), + zero(eltype(par)), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par, ) - if !isnothing(model.optimizer.local_algorithm) + if !isnothing(optim.local_algorithm) opt_local = construct_NLopt_problem( - model.optimizer.local_algorithm, - model.optimizer.local_options, + optim.local_algorithm, + optim.local_options, length(start_params), ) opt.local_optimizer = opt_local From 23e226500c217932a543f758c3cc0cabe454766c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 14:46:34 -0800 Subject: [PATCH 21/71] SEMProximalOptExt for Proximal opt --- Project.toml | 4 ++ ext/SEMProximalOptExt.jl | 15 ++++++++ ext/diff/Proximal.jl | 39 +++++++++++++++++++ ext/optimizer/ProximalAlgorithms.jl | 59 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 ext/SEMProximalOptExt.jl create mode 100644 ext/diff/Proximal.jl create mode 100644 ext/optimizer/ProximalAlgorithms.jl diff --git a/Project.toml b/Project.toml index 21bd43814..1bd335f19 100644 --- a/Project.toml +++ b/Project.toml @@ -46,6 +46,10 @@ test = ["Test"] [weakdeps] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" [extensions] SEMNLOptExt = "NLopt" +SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt.jl new file mode 100644 index 000000000..fb9f3c410 --- /dev/null +++ b/ext/SEMProximalOptExt.jl @@ -0,0 +1,15 @@ +module SEMProximalOptExt + +using StructuralEquationModels +using ProximalCore, ProximalAlgorithms, ProximalOperators + +export SemOptimizerProximal + +SEM = StructuralEquationModels + +#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) + +include("diff/Proximal.jl") +include("optimizer/ProximalAlgorithms.jl") + +end diff --git a/ext/diff/Proximal.jl b/ext/diff/Proximal.jl new file mode 100644 index 000000000..9c84c725a --- /dev/null +++ b/ext/diff/Proximal.jl @@ -0,0 +1,39 @@ +mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} + algorithm::A + options::B + operator_g::C + operator_h::D +end + +SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) + +SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., +) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerProximal) = optimizer.options + +############################################################################ +### Pretty Printing +############################################################################ + +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl new file mode 100644 index 000000000..379b0a209 --- /dev/null +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -0,0 +1,59 @@ +## connect do ProximalAlgorithms.jl as backend +ProximalCore.gradient!(grad, model::AbstractSem, parameters) = + objective_gradient!(grad, model::AbstractSem, parameters) + +mutable struct ProximalResult + result::Any +end + +function SEM.sem_fit( + optim::SemOptimizerProximal, + model::AbstractSem, + start_params::AbstractVector; + kwargs..., +) + if isnothing(optim.operator_h) + solution, iterations = + optim.algorithm(x0 = start_params, f = model, g = optim.operator_g) + else + solution, iterations = optim.algorithm( + x0 = start_params, + f = model, + g = optim.operator_g, + h = optim.operator_h, + ) + end + + minimum = objective!(model, solution) + + optimization_result = Dict( + :minimum => minimum, + :iterations => iterations, + :algorithm => optim.algorithm, + :operator_g => optim.operator_g, + ) + + isnothing(optim.operator_h) || + push!(optimization_result, :operator_h => optim.operator_h) + + return SemFit( + minimum, + solution, + start_params, + model, + ProximalResult(optimization_result), + ) +end + +############################################################################################ +# pretty printing +############################################################################################ + +function Base.show(io::IO, result::ProximalResult) + print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n") + print(io, "No. evaluations: $(result.result[:iterations]) \n") + print(io, "Operator: $(nameof(typeof(result.result[:operator_g]))) \n") + if haskey(result.result, :operator_h) + print(io, "Second Operator: $(nameof(typeof(result.result[:operator_h]))) \n") + end +end From 8a98831cb10aa6f2287a2883d88b7a11f7dbc4de Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 16:22:27 -0800 Subject: [PATCH 22/71] merge diff/*.jl optimizer code into optimizer/*.jl --- ext/SEMNLOptExt.jl | 1 - ext/SEMProximalOptExt.jl | 1 - ext/diff/NLopt.jl | 124 ---------------------------- ext/diff/Proximal.jl | 39 --------- ext/optimizer/NLopt.jl | 123 ++++++++++++++++++++++++++- ext/optimizer/ProximalAlgorithms.jl | 61 ++++++++++++++ src/StructuralEquationModels.jl | 4 +- src/diff/optim.jl | 71 ---------------- src/{diff => optimizer}/Empty.jl | 0 src/optimizer/optim.jl | 74 +++++++++++++++++ 10 files changed, 258 insertions(+), 240 deletions(-) delete mode 100644 ext/diff/NLopt.jl delete mode 100644 ext/diff/Proximal.jl delete mode 100644 src/diff/optim.jl rename src/{diff => optimizer}/Empty.jl (100%) diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt.jl index dfc3bbb42..a727b82f1 100644 --- a/ext/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt.jl @@ -6,7 +6,6 @@ SEM = StructuralEquationModels export SemOptimizerNLopt, NLoptConstraint -include("diff/NLopt.jl") include("optimizer/NLopt.jl") end diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt.jl index fb9f3c410..e81760acb 100644 --- a/ext/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt.jl @@ -9,7 +9,6 @@ SEM = StructuralEquationModels #ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) -include("diff/Proximal.jl") include("optimizer/ProximalAlgorithms.jl") end diff --git a/ext/diff/NLopt.jl b/ext/diff/NLopt.jl deleted file mode 100644 index 8267cf4bc..000000000 --- a/ext/diff/NLopt.jl +++ /dev/null @@ -1,124 +0,0 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `NLopt.jl` as the optimization backend. - -# Constructor - - SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm -- `local_algorithm`: local optimization algorithm -- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints - -# Example -```julia -my_optimizer = SemOptimizerNLopt() - -# constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; - algorithm = :AUGLAG, - local_algorithm = :LD_LBFGS, - local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), -) -``` - -# Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerNLopt)` -- `local_algorithm(::SemOptimizerNLopt)` -- `options(::SemOptimizerNLopt)` -- `local_options(::SemOptimizerNLopt)` -- `equality_constraints(::SemOptimizerNLopt)` -- `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C -end - -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end - -Base.convert( - ::Type{NLoptConstraint}, - tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, -) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) - -############################################################################################ -### Constructor -############################################################################################ - -function SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - local_algorithm = nothing, - options = Dict{Symbol, Any}(), - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs..., -) - applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || - (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) && - !isa(inequality_constraints, NamedTuple) || - (inequality_constraints = [inequality_constraints]) - return SemOptimizerNLopt( - algorithm, - local_algorithm, - options, - local_options, - convert.(NLoptConstraint, equality_constraints), - convert.(NLoptConstraint, inequality_constraints), - ) -end - -SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) - -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = - optimizer - -############################################################################################ -### additional methods -############################################################################################ - -SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm -local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm -SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options -local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options -equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints -inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints diff --git a/ext/diff/Proximal.jl b/ext/diff/Proximal.jl deleted file mode 100644 index 9c84c725a..000000000 --- a/ext/diff/Proximal.jl +++ /dev/null @@ -1,39 +0,0 @@ -mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} - algorithm::A - options::B - operator_g::C - operator_h::D -end - -SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) - -SemOptimizerProximal(; - algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), - operator_g, - operator_h = nothing, - kwargs..., -) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) - -############################################################################################ -### Recommended methods -############################################################################################ - -SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = - optimizer - -############################################################################################ -### additional methods -############################################################################################ - -SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm -SEM.options(optimizer::SemOptimizerProximal) = optimizer.options - -############################################################################ -### Pretty Printing -############################################################################ - -function Base.show(io::IO, struct_inst::SemOptimizerProximal) - print_type_name(io, struct_inst) - print_field_types(io, struct_inst) -end diff --git a/ext/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl index 94da98361..959380292 100644 --- a/ext/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -1,6 +1,127 @@ ############################################################################################ -### connect to NLopt.jl as backend +### Types ############################################################################################ +""" +Connects to `NLopt.jl` as the optimization backend. + +# Constructor + + SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + options = Dict{Symbol, Any}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `local_algorithm`: local optimization algorithm +- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm +- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints +- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints + +# Example +```julia +my_optimizer = SemOptimizerNLopt() + +# constrained optimization with augmented lagrangian +my_constrained_optimizer = SemOptimizerNLopt(; + algorithm = :AUGLAG, + local_algorithm = :LD_LBFGS, + local_options = Dict(:ftol_rel => 1e-6), + inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), +) +``` + +# Usage +All algorithms and options from the NLopt library are available, for more information see +the NLopt.jl package and the NLopt online documentation. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerNLopt)` +- `local_algorithm(::SemOptimizerNLopt)` +- `options(::SemOptimizerNLopt)` +- `local_options(::SemOptimizerNLopt)` +- `equality_constraints(::SemOptimizerNLopt)` +- `inequality_constraints(::SemOptimizerNLopt)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} + algorithm::A + local_algorithm::A2 + options::B + local_options::B2 + equality_constraints::C + inequality_constraints::C +end + +Base.@kwdef struct NLoptConstraint + f::Any + tol = 0.0 +end + +Base.convert( + ::Type{NLoptConstraint}, + tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, +) where {F, T} = NLoptConstraint(tuple.f, tuple.tol) + +############################################################################################ +### Constructor +############################################################################################ + +function SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + local_algorithm = nothing, + options = Dict{Symbol, Any}(), + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs..., +) + applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || + (equality_constraints = [equality_constraints]) + applicable(iterate, inequality_constraints) && + !isa(inequality_constraints, NamedTuple) || + (inequality_constraints = [inequality_constraints]) + return SemOptimizerNLopt( + algorithm, + local_algorithm, + options, + local_options, + convert.(NLoptConstraint, equality_constraints), + convert.(NLoptConstraint, inequality_constraints), + ) +end + +SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm +local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm +SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options +local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options +equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints +inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints mutable struct NLoptResult result::Any diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl index 379b0a209..8d7cc5b2d 100644 --- a/ext/optimizer/ProximalAlgorithms.jl +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -1,3 +1,64 @@ +############################################################################################ +### Types +############################################################################################ +""" +Connects to `ProximalAlgorithms.jl` as the optimization backend. + +# Constructor + + SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `operator_g`: gradient of the objective function +- `operator_h`: optional hessian of the objective function +""" +mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} + algorithm::A + options::B + operator_g::C + operator_h::D +end + +SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) + +SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + options = Dict{Symbol, Any}(), + operator_g, + operator_h = nothing, + kwargs..., +) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = + optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerProximal) = optimizer.options + +############################################################################ +### Pretty Printing +############################################################################ + +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end + ## connect do ProximalAlgorithms.jl as backend ProximalCore.gradient!(grad, model::AbstractSem, parameters) = objective_gradient!(grad, model::AbstractSem, parameters) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ec2abf31c..ca1ae61f0 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -60,10 +60,8 @@ include("loss/regularization/ridge.jl") include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer -include("diff/optim.jl") -include("diff/Empty.jl") -# optimizer include("optimizer/documentation.jl") +include("optimizer/Empty.jl") include("optimizer/optim.jl") # helper functions include("additional_functions/helper.jl") diff --git a/src/diff/optim.jl b/src/diff/optim.jl deleted file mode 100644 index 5b8845275..000000000 --- a/src/diff/optim.jl +++ /dev/null @@ -1,71 +0,0 @@ -############################################################################################ -### Types and Constructor -############################################################################################ -""" -Connects to `Optim.jl` as the optimization backend. - -# Constructor - - SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Optim.Options`: options for the optimization algorithm - -# Usage -All algorithms and options from the Optim.jl library are available, for more information see -the Optim.jl online documentation. - -# Examples -```julia -my_optimizer = SemOptimizerOptim() - -# hessian based optimization with backtracking linesearch and modified initial step size -using Optim, LineSearches - -my_newton_optimizer = SemOptimizerOptim( - algorithm = Newton( - ;linesearch = BackTracking(order=3), - alphaguess = InitialHagerZhang() - ) -) -``` - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerOptim)` -- `options(::SemOptimizerOptim)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} - algorithm::A - options::B -end - -SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) - -SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), - kwargs..., -) = SemOptimizerOptim(algorithm, options) - -############################################################################################ -### Recommended methods -############################################################################################ - -update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer - -############################################################################################ -### additional methods -############################################################################################ - -algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm -options(optimizer::SemOptimizerOptim) = optimizer.options diff --git a/src/diff/Empty.jl b/src/optimizer/Empty.jl similarity index 100% rename from src/diff/Empty.jl rename to src/optimizer/Empty.jl diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 19623b965..4031f2e4a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -1,5 +1,79 @@ ## connect to Optim.jl as backend +############################################################################################ +### Types and Constructor +############################################################################################ +""" + SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} + +Connects to `Optim.jl` as the optimization backend. + +# Constructor + + SemOptimizerOptim(; + algorithm = LBFGS(), + options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Optim.Options`: options for the optimization algorithm + +# Usage +All algorithms and options from the Optim.jl library are available, for more information see +the Optim.jl online documentation. + +# Examples +```julia +my_optimizer = SemOptimizerOptim() + +# hessian based optimization with backtracking linesearch and modified initial step size +using Optim, LineSearches + +my_newton_optimizer = SemOptimizerOptim( + algorithm = Newton( + ;linesearch = BackTracking(order=3), + alphaguess = InitialHagerZhang() + ) +) +``` + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerOptim)` +- `options(::SemOptimizerOptim)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} + algorithm::A + options::B +end + +SemOptimizer{:Optim}(args...; kwargs...) = SemOptimizerOptim(args...; kwargs...) + +SemOptimizerOptim(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerOptim(algorithm, options) + +############################################################################################ +### Recommended methods +############################################################################################ + +update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer + +############################################################################################ +### additional methods +############################################################################################ + +algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm +options(optimizer::SemOptimizerOptim) = optimizer.options + function SemFit( optimization_result::Optim.MultivariateOptimizationResults, model::AbstractSem, From 9e2672dcb6aabc9634f13e3e566e3a975775a782 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 20 Dec 2024 15:05:44 -0800 Subject: [PATCH 23/71] Optim: document u/l bounds --- src/optimizer/optim.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 4031f2e4a..cec37a77a 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -40,6 +40,16 @@ my_newton_optimizer = SemOptimizerOptim( # Extended help +## Constrained optimization + +When using the `Fminbox` or `SAMIN` constrained optimization algorithms, +the vector or dictionary of lower and upper bounds for each model parameter can be specified +via `lower_bounds` and `upper_bounds` keyword arguments. +Alternatively, the `lower_bound` and `upper_bound` keyword arguments can be used to specify +the default bound for all non-variance model parameters, +and the `variance_lower_bound` and `variance_upper_bound` keyword -- +for the variance parameters (the diagonal of the *S* matrix). + ## Interfaces - `algorithm(::SemOptimizerOptim)` - `options(::SemOptimizerOptim)` From d6188981543669d649b668394782638091665013 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 19 Dec 2024 15:00:34 +0100 Subject: [PATCH 24/71] remove unused options field from Proximal optimizer --- ext/optimizer/ProximalAlgorithms.jl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl index 8d7cc5b2d..13debf79d 100644 --- a/ext/optimizer/ProximalAlgorithms.jl +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -8,33 +8,29 @@ Connects to `ProximalAlgorithms.jl` as the optimization backend. SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), operator_g, operator_h = nothing, kwargs..., # Arguments - `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm - `operator_g`: gradient of the objective function - `operator_h`: optional hessian of the objective function """ -mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} algorithm::A - options::B - operator_g::C - operator_h::D + operator_g::B + operator_h::C end SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) SemOptimizerProximal(; algorithm = ProximalAlgorithms.PANOC(), - options = Dict{Symbol, Any}(), operator_g, operator_h = nothing, kwargs..., -) = SemOptimizerProximal(algorithm, options, operator_g, operator_h) +) = SemOptimizerProximal(algorithm, operator_g, operator_h) ############################################################################################ ### Recommended methods @@ -48,7 +44,6 @@ SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwar ############################################################################################ SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm -SEM.options(optimizer::SemOptimizerProximal) = optimizer.options ############################################################################ ### Pretty Printing From d055c78b430ed24e6f9aceb2be3fc96672c67750 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 21:38:12 -0800 Subject: [PATCH 25/71] decouple optimizer from Sem model Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/additional_functions/simulation.jl | 15 +--- .../start_val/start_fabin3.jl | 12 +-- .../start_val/start_simple.jl | 10 +-- src/frontend/fit/fitmeasures/chi2.jl | 7 +- src/frontend/fit/fitmeasures/minus2ll.jl | 19 ++-- src/frontend/specification/Sem.jl | 44 +++------- src/optimizer/documentation.jl | 26 +++--- src/types.jl | 62 ++++--------- test/examples/political_democracy/by_parts.jl | 65 +++++++------- .../political_democracy/constraints.jl | 19 +--- .../political_democracy/constructor.jl | 86 ++++++------------- .../recover_parameters_twofact.jl | 7 +- test/unit_tests/model.jl | 1 - 13 files changed, 130 insertions(+), 243 deletions(-) diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index f1e41f360..0b2626b15 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -47,7 +47,6 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = observed(model), imply(model), loss(model), - optimizer(model), new_observed; kwargs..., ) @@ -57,7 +56,6 @@ function swap_observed( old_observed, imply, loss, - optimizer, new_observed::SemObserved; kwargs..., ) @@ -68,7 +66,6 @@ function swap_observed( kwargs[:old_observed_type] = typeof(old_observed) kwargs[:imply_type] = typeof(imply) kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions] - kwargs[:optimizer_type] = typeof(optimizer) # update imply imply = update_observed(imply, new_observed; kwargs...) @@ -79,16 +76,12 @@ function swap_observed( loss = update_observed(loss, new_observed; kwargs...) kwargs[:loss] = loss - # update optimizer - optimizer = update_observed(optimizer, new_observed; kwargs...) - #new_imply = update_observed(model.imply, new_observed; kwargs...) return Sem( new_observed, update_observed(model.imply, new_observed; kwargs...), update_observed(model.loss, new_observed; kwargs...), - update_observed(model.optimizer, new_observed; kwargs...), ) end @@ -120,18 +113,18 @@ rand(model, start_simple(model), 100) ``` """ function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, + model::AbstractSemSingle{O, I, L}, params, n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L, D} +) where {O, I <: Union{RAM, RAMSymbolic}, L} update!(EvaluationTargets{true, false, false}(), model.imply, model, params) return rand(model, n) end function Distributions.rand( - model::AbstractSemSingle{O, I, L, D}, + model::AbstractSemSingle{O, I, L}, n::Integer, -) where {O, I <: Union{RAM, RAMSymbolic}, L, D} +) where {O, I <: Union{RAM, RAMSymbolic}, L} if MeanStruct(model.imply) === NoMeanStruct data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) elseif MeanStruct(model.imply) === HasMeanStruct diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 53cf7cff6..dd8d61fd9 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -8,21 +8,15 @@ function start_fabin3 end # splice model and loss functions function start_fabin3(model::AbstractSemSingle; kwargs...) - return start_fabin3( - model.observed, - model.imply, - model.optimizer, - model.loss.functions..., - kwargs..., - ) + return start_fabin3(model.observed, model.imply, model.loss.functions..., kwargs...) end -function start_fabin3(observed, imply, optimizer, args...; kwargs...) +function start_fabin3(observed, imply, args...; kwargs...) return start_fabin3(imply.ram_matrices, obs_cov(observed), obs_mean(observed)) end # SemObservedMissing -function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; kwargs...) +function start_fabin3(observed::SemObservedMissing, imply, args...; kwargs...) if !observed.em_model.fitted em_mvn(observed; kwargs...) end diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 1f73a3583..1f16b094c 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -17,16 +17,10 @@ function start_simple end # Single Models ---------------------------------------------------------------------------- function start_simple(model::AbstractSemSingle; kwargs...) - return start_simple( - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs..., - ) + return start_simple(model.observed, model.imply, model.loss.functions...; kwargs...) end -function start_simple(observed, imply, optimizer, args...; kwargs...) +function start_simple(observed, imply, args...; kwargs...) return start_simple(imply.ram_matrices; kwargs...) end diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index df1027bd6..12bc1d880 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -14,21 +14,20 @@ function χ² end sem_fit, sem_fit.model.observed, sem_fit.model.imply, - sem_fit.model.optimizer, sem_fit.model.loss.functions..., ) # RAM + SemML -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = +χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = (nsamples(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed)) # bollen, p. 115, only correct for GLS weight matrix -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = +χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = (nsamples(sem_fit) - 1) * sem_fit.minimum # FIML -function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, optimizer, loss_ml::SemFIML) +function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, loss_ml::SemFIML) ll_H0 = minus2ll(sem_fit) ll_H1 = minus2ll(observed) chi2 = ll_H0 - ll_H1 diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 88948d4d4..54a4ce12d 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -16,30 +16,21 @@ minus2ll( sem_fit, sem_fit.model.observed, sem_fit.model.imply, - sem_fit.model.optimizer, sem_fit.model.loss.functions..., ) -minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) = - minus2ll(sem_fit.minimum, obs, imp, optimizer, args...) +minus2ll(sem_fit::SemFit, obs, imp, args...) = minus2ll(sem_fit.minimum, obs, imp, args...) # SemML ------------------------------------------------------------------------------------ -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = +minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemML) = nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs)) # WLS -------------------------------------------------------------------------------------- -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = - missing +minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll( - minimum::Number, - observed, - imp::Union{RAM, RAMSymbolic}, - optimizer, - loss_ml::SemFIML, -) +function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML) F = minimum F *= nsamples(observed) F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars) @@ -117,7 +108,7 @@ end ############################################################################################ minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.imply, model.optimizer, model.loss.functions...) + minus2ll(minimum, model.observed, model.imply, model.loss.functions...) function minus2ll( sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}, diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 758bc073d..741d5f3c6 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -6,16 +6,15 @@ function Sem(; observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, - optimizer::D = SemOptimizerOptim, kwargs..., -) where {O, I, L, D} +) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) + set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) + observed, imply, loss = get_fields!(kwdict, observed, imply, loss) - sem = Sem(observed, imply, loss, optimizer) + sem = Sem(observed, imply, loss) return sem end @@ -59,27 +58,19 @@ Returns the loss part of a model. """ loss(model::AbstractSemSingle) = model.loss -""" - optimizer(model::AbstractSemSingle) -> SemOptimizer - -Returns the optimizer part of a model. -""" -optimizer(model::AbstractSemSingle) = model.optimizer - function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, - optimizer::D = SemOptimizerOptim, kwargs..., -) where {O, I, L, D} +) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) + set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) + observed, imply, loss = get_fields!(kwdict, observed, imply, loss) - sem = SemFiniteDiff(observed, imply, loss, optimizer) + sem = SemFiniteDiff(observed, imply, loss) return sem end @@ -88,7 +79,7 @@ end # functions ############################################################################################ -function set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, D) +function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) kwargs[:observed_type] = O <: Type ? observed : typeof(observed) kwargs[:imply_type] = I <: Type ? imply : typeof(imply) if loss isa SemLoss @@ -102,11 +93,10 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, else kwargs[:loss_types] = [loss isa SemLossFunction ? typeof(loss) : loss] end - kwargs[:optimizer_type] = D <: Type ? optimizer : typeof(optimizer) end # construct Sem fields -function get_fields!(kwargs, observed, imply, loss, optimizer) +function get_fields!(kwargs, observed, imply, loss) # observed if !isa(observed, SemObserved) observed = observed(; kwargs...) @@ -125,12 +115,7 @@ function get_fields!(kwargs, observed, imply, loss, optimizer) loss = get_SemLoss(loss; kwargs...) kwargs[:loss] = loss - # optimizer - if !isa(optimizer, SemOptimizer) - optimizer = optimizer(; kwargs...) - end - - return observed, imply, loss, optimizer + return observed, imply, loss end # construct loss field @@ -167,7 +152,7 @@ end print(io, "Sem{$(nameof(O)), $(nameof(I)), $lossfuntypes, $(nameof(D))}") end =# -function Base.show(io::IO, sem::Sem{O, I, L, D}) where {O, I, L, D} +function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) lossfuntypes = " " .* lossfuntypes .* ("\n") print(io, "Structural Equation Model \n") @@ -176,10 +161,9 @@ function Base.show(io::IO, sem::Sem{O, I, L, D}) where {O, I, L, D} print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") print(io, " imply: $(nameof(I)) \n") - print(io, " optimizer: $(nameof(D)) \n") end -function Base.show(io::IO, sem::SemFiniteDiff{O, I, L, D}) where {O, I, L, D} +function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} lossfuntypes = @. string(nameof(typeof(sem.loss.functions))) lossfuntypes = " " .* lossfuntypes .* ("\n") print(io, "Structural Equation Model : Finite Diff Approximation\n") @@ -188,7 +172,6 @@ function Base.show(io::IO, sem::SemFiniteDiff{O, I, L, D}) where {O, I, L, D} print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") print(io, " imply: $(nameof(I)) \n") - print(io, " optimizer: $(nameof(D)) \n") end function Base.show(io::IO, loss::SemLoss) @@ -211,7 +194,6 @@ function Base.show(io::IO, models::SemEnsemble) print(io, "SemEnsemble \n") print(io, "- Number of Models: $(models.n) \n") print(io, "- Weights: $(round.(models.weights, digits = 2)) \n") - print(io, "- optimizer: $(nameof(typeof(optimizer(models)))) \n") print(io, "\n", "Models: \n") print(io, "===============================================", "\n") diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index cf6aaa312..c6669aa12 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -1,16 +1,22 @@ """ - sem_fit(model::AbstractSem; start_val = start_val, kwargs...) + sem_fit([optim::SemOptimizer], model::AbstractSem; + [engine::Symbol], start_val = start_val, kwargs...) Return the fitted `model`. # Arguments +- `optim`: [`SemOptimizer`](@ref) to use for fitting. + If omitted, a new optimizer is constructed as `SemOptimizer(; engine, kwargs...)`. - `model`: `AbstractSem` to fit +- `engine`: the optimization engine to use, default is `:Optim` - `start_val`: a vector or a dictionary of starting parameter values, or function to compute them (1) -- `kwargs...`: keyword arguments, passed to starting value functions +- `kwargs...`: keyword arguments, passed to optimization engine constructor and + `start_val` function (1) available functions are `start_fabin3`, `start_simple` and `start_partable`. -For more information, we refer to the individual documentations and the online documentation on [Starting values](@ref). +For more information, we refer to the individual documentations and +the online documentation on [Starting values](@ref). # Examples ```julia @@ -20,20 +26,20 @@ sem_fit( start_covariances_latent = 0.5) ``` """ -function sem_fit end - -# dispatch on optimizer -function sem_fit(model::AbstractSem; start_val = nothing, kwargs...) +function sem_fit(optim::SemOptimizer, model::AbstractSem; start_val = nothing, kwargs...) start_params = prepare_start_params(start_val, model; kwargs...) @assert start_params isa AbstractVector @assert length(start_params) == nparams(model) - sem_fit(model.optimizer, model, start_params; kwargs...) + sem_fit(optim, model, start_params; kwargs...) end +sem_fit(model::AbstractSem; engine::Symbol = :Optim, start_val = nothing, kwargs...) = + sem_fit(SemOptimizer(; engine, kwargs...), model; start_val, kwargs...) + # fallback method -sem_fit(optimizer::SemOptimizer, model::AbstractSem, start_params; kwargs...) = - error("Optimizer $(optimizer) support not implemented.") +sem_fit(optim::SemOptimizer, model::AbstractSem, start_params; kwargs...) = + error("Optimizer $(optim) support not implemented.") # FABIN3 is the default method for single models prepare_start_params(start_val::Nothing, model::AbstractSemSingle; kwargs...) = diff --git a/src/types.jl b/src/types.jl index 90b648ac8..cfe916d9e 100644 --- a/src/types.jl +++ b/src/types.jl @@ -4,8 +4,8 @@ "Most abstract supertype for all SEMs" abstract type AbstractSem end -"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss` and `optimizer`" -abstract type AbstractSemSingle{O, I, L, D} <: AbstractSem end +"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss`" +abstract type AbstractSemSingle{O, I, L} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end @@ -116,73 +116,66 @@ abstract type SemImply end abstract type SemImplySymbolic <: SemImply end """ - Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) + Sem(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) Constructor for the basic `Sem` type. -All additional kwargs are passed down to the constructors for the observed, imply, loss and optimizer fields. +All additional kwargs are passed down to the constructors for the observed, imply, and loss fields. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. - `imply`: object of subtype `SemImply` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). - `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). """ -mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss, D <: SemOptimizer} <: - AbstractSemSingle{O, I, L, D} +mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O imply::I loss::L - optimizer::D end ############################################################################################ # automatic differentiation ############################################################################################ """ - SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) + SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) -Constructor for `SemFiniteDiff`. -All additional kwargs are passed down to the constructors for the observed, imply, loss and optimizer fields. +A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with +finite difference approximation. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. - `imply`: object of subtype `SemImply` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). - `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). """ -struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss, D <: SemOptimizer} <: - AbstractSemSingle{O, I, L, D} +struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O imply::I loss::L - optimizer::D end ############################################################################################ # ensemble models ############################################################################################ """ - (1) SemEnsemble(models..., optimizer = SemOptimizerOptim, weights = nothing, kwargs...) + (1) SemEnsemble(models..., weights = nothing, kwargs...) - (2) SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) + (2) SemEnsemble(;specification, data, groups, column = :group, kwargs...) Constructor for ensemble models. (2) can be used to conveniently specify multigroup models. # Arguments - `models...`: `AbstractSem`s. -- `optimizer`: object of subtype `SemOptimizer` or a constructor. - `weights::Vector`: Weights for each model. Defaults to the number of observed data points. - `specification::EnsembleParameterTable`: Model specification. - `data::DataFrame`: Observed data. Must contain a `column` of type `Vector{Symbol}` that contains the group. @@ -195,19 +188,17 @@ Returns a SemEnsemble with fields - `n::Int`: Number of models. - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. -- `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). - `params::Vector`: Stores parameter labels and their position. """ -struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemCollection +struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, I} <: AbstractSemCollection n::N sems::T weights::V - optimizer::D params::I end # constructor from multiple models -function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) +function SemEnsemble(models...; weights = nothing, kwargs...) n = length(models) # default weights @@ -227,16 +218,11 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing end end - # optimizer - if !isa(optimizer, SemOptimizer) - optimizer = optimizer(; kwargs...) - end - - return SemEnsemble(n, models, weights, optimizer, params) + return SemEnsemble(n, models, weights, params) end # constructor from EnsembleParameterTable and data set -function SemEnsemble(;specification, data, groups, column = :group, optimizer = SemOptimizerOptim, kwargs...) +function SemEnsemble(; specification, data, groups, column = :group, kwargs...) if specification isa EnsembleParameterTable specification = convert(Dict{Symbol, RAMMatrices}, specification) end @@ -247,14 +233,10 @@ function SemEnsemble(;specification, data, groups, column = :group, optimizer = if iszero(nrow(data_group)) error("Your data does not contain any observations from group `$(group)`.") end - model = Sem(; - specification = ram_matrices, - data = data_group, - kwargs... - ) + model = Sem(; specification = ram_matrices, data = data_group, kwargs...) push!(models, model) end - return SemEnsemble(models...; optimizer = optimizer, weights = nothing, kwargs...) + return SemEnsemble(models...; weights = nothing, kwargs...) end params(ensemble::SemEnsemble) = ensemble.params @@ -277,12 +259,6 @@ models(ensemble::SemEnsemble) = ensemble.sems Returns the weights of an ensemble model. """ weights(ensemble::SemEnsemble) = ensemble.weights -""" - optimizer(ensemble::SemEnsemble) -> SemOptimizer - -Returns the optimizer part of an ensemble model. -""" -optimizer(ensemble::SemEnsemble) = ensemble.optimizer """ Base type for all SEM specifications. diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 87e5fb733..5e5244f91 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -29,23 +29,18 @@ optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_ml) -model_ls_sym = - Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls, optimizer_obj) +model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) -model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge), optimizer_obj) +model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge)) -model_constant = Sem(observed, imply_ram, SemLoss(ml, constant), optimizer_obj) +model_constant = Sem(observed, imply_ram, SemLoss(ml, constant)) -model_ml_weighted = Sem( - observed, - imply_ram, - SemLoss(ml; loss_weights = [nsamples(model_ml)]), - optimizer_obj, -) +model_ml_weighted = + Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) ############################################################################################ ### test gradients @@ -75,7 +70,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml", "ml"]) for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(model) + solution = sem_fit(optimizer_obj, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -84,9 +79,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(model_ridge) - solution_ml = sem_fit(model_ml) - # solution_ridge_id = sem_fit(model_ridge_id) + solution_ridge = sem_fit(optimizer_obj, model_ridge) + solution_ml = sem_fit(optimizer_obj, model_ml) + # solution_ridge_id = sem_fit(optimizer_obj, model_ridge_id) @test solution_ridge.minimum < solution_ml.minimum + 1 end @@ -102,8 +97,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(model_ml) - solution_ml_weighted = sem_fit(model_ml_weighted) + solution_ml = sem_fit(optimizer_obj, model_ml) + solution_ml_weighted = sem_fit(optimizer_obj, model_ml_weighted) @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 @test nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml) ≈ StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 @@ -114,7 +109,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -128,7 +123,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(model_ls_sym) + solution_ls = sem_fit(optimizer_obj, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -167,10 +162,9 @@ if opt_engine == :Optim imply_sym_hessian = RAMSymbolic(specification = spec, hessian = true) - model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls, optimizer_obj) + model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls) - model_ml = - Sem(observed, imply_sym_hessian, loss_ml, SemOptimizerOptim(algorithm = Newton())) + model_ml = Sem(observed, imply_sym_hessian, loss_ml) @testset "ml_hessians" begin test_hessian(model_ml, start_test; atol = 1e-4) @@ -181,13 +175,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(model_ml) + solution = sem_fit(optimizer_obj, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin - solution = sem_fit(model_ls) + solution = sem_fit(optimizer_obj, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -224,16 +218,15 @@ loss_wls = SemLoss(wls) optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_ml) model_ls = Sem( observed, RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true), loss_wls, - optimizer_obj, ) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) ############################################################################################ ### test gradients @@ -260,7 +253,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ls", "ml"] .* "_mean" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(model) + solution = sem_fit(optimizer_obj, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -273,7 +266,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -291,7 +284,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(model_ls) + solution_ls = sem_fit(optimizer_obj, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -321,9 +314,9 @@ fiml = SemFIML(observed = observed, specification = spec_mean) loss_fiml = SemLoss(fiml) -model_ml = Sem(observed, imply_ram, loss_fiml, optimizer_obj) +model_ml = Sem(observed, imply_ram, loss_fiml) -model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml, optimizer_obj) +model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml) ############################################################################################ ### test gradients @@ -342,13 +335,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(model_ml) + solution = sem_fit(optimizer_obj, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(model_ml_sym) + solution = sem_fit(optimizer_obj, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -358,7 +351,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(optimizer_obj, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index 47f27582a..ef5692f27 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -30,33 +30,22 @@ constrained_optimizer = SemOptimizer(; inequality_constraints = (f = ineq_constraint, tol = 0.0), ) -model_ml_constrained = - Sem(specification = spec, data = dat, optimizer = constrained_optimizer) - -solution_constrained = sem_fit(model_ml_constrained) - # NLopt option setting --------------------------------------------------------------------- -model_ml_maxeval = Sem( - specification = spec, - data = dat, - optimizer = SemOptimizer, - engine = :NLopt, - options = Dict(:maxeval => 10), -) - ############################################################################################ ### test solution ############################################################################################ @testset "ml_solution_maxeval" begin - solution_maxeval = sem_fit(model_ml_maxeval) + solution_maxeval = sem_fit(model_ml, engine = :NLopt, options = Dict(:maxeval => 10)) + @test solution_maxeval.optimization_result.problem.numevals == 10 @test solution_maxeval.optimization_result.result[3] == :MAXEVAL_REACHED end @testset "ml_solution_constrained" begin - solution_constrained = sem_fit(model_ml_constrained) + solution_constrained = sem_fit(constrained_optimizer, model_ml) + @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) @test all(abs.(solution_constrained.solution) .< 10) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 5ed576dc1..cba86aef0 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -7,7 +7,7 @@ using Random, NLopt semoptimizer = SemOptimizer(engine = opt_engine) -model_ml = Sem(specification = spec, data = dat, optimizer = semoptimizer) +model_ml = Sem(specification = spec, data = dat) @test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( @@ -15,20 +15,12 @@ model_ml_cov = Sem( observed = SemObservedCovariance, obs_cov = cov(Matrix(dat)), obs_colnames = Symbol.(names(dat)), - optimizer = semoptimizer, nsamples = 75, ) -model_ls_sym = Sem( - specification = spec, - data = dat, - imply = RAMSymbolic, - loss = SemWLS, - optimizer = semoptimizer, -) +model_ls_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic, loss = SemWLS) -model_ml_sym = - Sem(specification = spec, data = dat, imply = RAMSymbolic, optimizer = semoptimizer) +model_ml_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic) model_ridge = Sem( specification = spec, @@ -36,7 +28,6 @@ model_ridge = Sem( loss = (SemML, SemRidge), α_ridge = 0.001, which_ridge = 16:20, - optimizer = semoptimizer, ) model_constant = Sem( @@ -44,15 +35,10 @@ model_constant = Sem( data = dat, loss = (SemML, SemConstant), constant_loss = 3.465, - optimizer = semoptimizer, ) -model_ml_weighted = Sem( - specification = partable, - data = dat, - loss_weights = (nsamples(model_ml),), - optimizer = semoptimizer, -) +model_ml_weighted = + Sem(specification = partable, data = dat, loss_weights = (nsamples(model_ml),)) ############################################################################################ ### test gradients @@ -89,7 +75,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml", "ml" for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution" begin - solution = sem_fit(model) + solution = sem_fit(semoptimizer, model) update_estimate!(partable, solution) test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end @@ -98,9 +84,9 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) end @testset "ridge_solution" begin - solution_ridge = sem_fit(model_ridge) - solution_ml = sem_fit(model_ml) - # solution_ridge_id = sem_fit(model_ridge_id) + solution_ridge = sem_fit(semoptimizer, model_ridge) + solution_ml = sem_fit(semoptimizer, model_ml) + # solution_ridge_id = sem_fit(semoptimizer, model_ridge_id) @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 end @@ -116,8 +102,8 @@ end end @testset "ml_solution_weighted" begin - solution_ml = sem_fit(model_ml) - solution_ml_weighted = sem_fit(model_ml_weighted) + solution_ml = sem_fit(semoptimizer, model_ml) + solution_ml_weighted = sem_fit(semoptimizer, model_ml_weighted) @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) @test isapprox( nsamples(model_ml) * StructuralEquationModels.minimum(solution_ml), @@ -131,7 +117,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) update_se_hessian!(partable, solution_ml) @@ -145,7 +131,7 @@ end end @testset "fitmeasures/se_ls" begin - solution_ls = sem_fit(model_ls_sym) + solution_ls = sem_fit(semoptimizer, model_ls_sym) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -196,8 +182,8 @@ end obs_colnames = colnames, ) # fit models - sol_ml = solution(sem_fit(model_ml_new)) - sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -239,13 +225,13 @@ if opt_engine == :Optim end @testset "ml_solution_hessian" begin - solution = sem_fit(model_ml) + solution = sem_fit(semoptimizer, model_ml) update_estimate!(partable, solution) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin - solution = sem_fit(model_ls) + solution = sem_fit(semoptimizer, model_ls) update_estimate!(partable, solution) test_estimates( partable, @@ -268,15 +254,9 @@ model_ls = Sem( imply = RAMSymbolic, loss = SemWLS, meanstructure = true, - optimizer = semoptimizer, ) -model_ml = Sem( - specification = spec_mean, - data = dat, - meanstructure = true, - optimizer = semoptimizer, -) +model_ml = Sem(specification = spec_mean, data = dat, meanstructure = true) model_ml_cov = Sem( specification = spec_mean, @@ -285,18 +265,11 @@ model_ml_cov = Sem( obs_mean = vcat(mean(Matrix(dat), dims = 1)...), obs_colnames = Symbol.(names(dat)), meanstructure = true, - optimizer = semoptimizer, nsamples = 75, ) -model_ml_sym = Sem( - specification = spec_mean, - data = dat, - imply = RAMSymbolic, - meanstructure = true, - start_val = start_test_mean, - optimizer = semoptimizer, -) +model_ml_sym = + Sem(specification = spec_mean, data = dat, imply = RAMSymbolic, meanstructure = true) ############################################################################################ ### test gradients @@ -323,7 +296,7 @@ solution_names = Symbol.("parameter_estimates_" .* ["ml", "ml", "ls", "ml"] .* " for (model, name, solution_name) in zip(models, model_names, solution_names) try @testset "$(name)_solution_mean" begin - solution = sem_fit(model) + solution = sem_fit(semoptimizer, model) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end @@ -336,7 +309,7 @@ end ############################################################################################ @testset "fitmeasures/se_ml_mean" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; @@ -354,7 +327,7 @@ end end @testset "fitmeasures/se_ls_mean" begin - solution_ls = sem_fit(model_ls) + solution_ls = sem_fit(semoptimizer, model_ls) fm = fit_measures(solution_ls) test_fitmeasures( fm, @@ -408,8 +381,8 @@ end meanstructure = true, ) # fit models - sol_ml = solution(sem_fit(model_ml_new)) - sol_ml_sym = solution(sem_fit(model_ml_sym_new)) + sol_ml = solution(sem_fit(semoptimizer, model_ml_new)) + sol_ml_sym = solution(sem_fit(semoptimizer, model_ml_sym_new)) # check solution @test maximum(abs.(sol_ml - params)) < 0.01 @test maximum(abs.(sol_ml_sym - params)) < 0.01 @@ -425,7 +398,6 @@ model_ml = Sem( data = dat_missing, observed = SemObservedMissing, loss = SemFIML, - optimizer = semoptimizer, meanstructure = true, ) @@ -435,8 +407,6 @@ model_ml_sym = Sem( observed = SemObservedMissing, imply = RAMSymbolic, loss = SemFIML, - start_val = start_test_mean, - optimizer = semoptimizer, meanstructure = true, ) @@ -457,13 +427,13 @@ end ############################################################################################ @testset "fiml_solution" begin - solution = sem_fit(model_ml) + solution = sem_fit(semoptimizer, model_ml) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin - solution = sem_fit(model_ml_sym) + solution = sem_fit(semoptimizer, model_ml_sym) update_estimate!(partable_mean, solution) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @@ -473,7 +443,7 @@ end ############################################################################################ @testset "fitmeasures/se_fiml" begin - solution_ml = sem_fit(model_ml) + solution_ml = sem_fit(semoptimizer, model_ml) test_fitmeasures( fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 89c1225e2..4b968bc49 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -65,13 +65,14 @@ semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) +model_ml = Sem(semobserved, imply_ml, loss_ml) +objective!(model_ml, true_val) + optimizer = SemOptimizerOptim( BFGS(; linesearch = BackTracking(order = 3), alphaguess = InitialHagerZhang()),# m = 100), Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), ) -model_ml = Sem(semobserved, imply_ml, loss_ml, optimizer) -objective!(model_ml, true_val) -solution_ml = sem_fit(model_ml) +solution_ml = sem_fit(optimizer, model_ml) @test true_val ≈ solution(solution_ml) atol = 0.05 diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index e13327642..bf44091d2 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -59,7 +59,6 @@ end @test model isa Sem @test @inferred(imply(model)) isa implytype @test @inferred(observed(model)) isa SemObserved - @test @inferred(optimizer(model)) isa SemOptimizer test_vars_api(model, ram_matrices) test_params_api(model, ram_matrices) From 71bced74dfbddc449dea51c2cbad394cff429b4e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Nov 2024 22:58:09 -0800 Subject: [PATCH 26/71] fix inequality constraints test NLopt minimum was 18.11, below what the test expected --- test/examples/political_democracy/constraints.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index ef5692f27..fb2116023 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -30,6 +30,8 @@ constrained_optimizer = SemOptimizer(; inequality_constraints = (f = ineq_constraint, tol = 0.0), ) +@test constrained_optimizer isa SemOptimizer{:NLopt} + # NLopt option setting --------------------------------------------------------------------- ############################################################################################ @@ -49,6 +51,6 @@ end @test solution_constrained.solution[31] * solution_constrained.solution[30] >= (0.6 - 1e-8) @test all(abs.(solution_constrained.solution) .< 10) - @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED skip = true - @test abs(solution_constrained.minimum - 21.21) < 0.01 + @test solution_constrained.optimization_result.result[3] == :FTOL_REACHED + @test solution_constrained.minimum <= 21.21 + 0.01 end From 928af39fee7f4c894f293cb8d9da39c46edb018b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Fri, 20 Dec 2024 09:02:16 -0800 Subject: [PATCH 27/71] add ProximalSEM tests --- test/Project.toml | 3 ++ test/examples/examples.jl | 3 ++ test/examples/proximal/l0.jl | 67 ++++++++++++++++++++++++++++++ test/examples/proximal/lasso.jl | 64 ++++++++++++++++++++++++++++ test/examples/proximal/proximal.jl | 9 ++++ test/examples/proximal/ridge.jl | 61 +++++++++++++++++++++++++++ 6 files changed, 207 insertions(+) create mode 100644 test/examples/proximal/l0.jl create mode 100644 test/examples/proximal/lasso.jl create mode 100644 test/examples/proximal/proximal.jl create mode 100644 test/examples/proximal/ridge.jl diff --git a/test/Project.toml b/test/Project.toml index 5867c1f40..14bd0bece 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -9,6 +9,9 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/test/examples/examples.jl b/test/examples/examples.jl index a1e0f2c28..e088ffa92 100644 --- a/test/examples/examples.jl +++ b/test/examples/examples.jl @@ -9,3 +9,6 @@ end @safetestset "Multigroup" begin include("multigroup/multigroup.jl") end +@safetestset "Proximal" begin + include("proximal/proximal.jl") +end diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl new file mode 100644 index 000000000..e8874fd51 --- /dev/null +++ b/test/examples/proximal/l0.jl @@ -0,0 +1,67 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph; latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use l0 from ProximalSEM +# regularized +prox_operator = + SlicedSeparableSum((NormL0(0.0), NormL0(0.02)), ([vcat(1:15, 21:31)], [12:20])) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) + +@testset "l0 | solution_unregularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 +end + +# regularized +prox_operator = SlicedSeparableSum((NormL0(0.0), NormL0(100.0)), ([1:30], [31])) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = prox_operator) + +@testset "l0 | solution_regularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test solution(fit_prox)[31] == 0.0 + @test abs( + StructuralEquationModels.minimum(fit_prox) - StructuralEquationModels.minimum(fit), + ) < 1.0 +end diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl new file mode 100644 index 000000000..31a4073f9 --- /dev/null +++ b/test/examples/proximal/lasso.jl @@ -0,0 +1,64 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use lasso from ProximalSEM +λ = zeros(31) + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) + +@testset "lasso | solution_unregularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test maximum(abs.(solution(fit) - solution(fit_prox))) < 0.002 +end + +λ = zeros(31); +λ[16:20] .= 0.02; + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +fit_prox = sem_fit(model_prox, engine = :Proximal, operator_g = NormL1(λ)) + +@testset "lasso | solution_regularized" begin + @test fit_prox.optimization_result.result[:iterations] < 1000 + @test all(solution(fit_prox)[16:20] .< solution(fit)[16:20]) + @test StructuralEquationModels.minimum(fit_prox) - + StructuralEquationModels.minimum(fit) < 0.03 +end diff --git a/test/examples/proximal/proximal.jl b/test/examples/proximal/proximal.jl new file mode 100644 index 000000000..40e72a1ef --- /dev/null +++ b/test/examples/proximal/proximal.jl @@ -0,0 +1,9 @@ +@testset "Ridge" begin + include("ridge.jl") +end +@testset "Lasso" begin + include("lasso.jl") +end +@testset "L0" begin + include("l0.jl") +end diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl new file mode 100644 index 000000000..120910234 --- /dev/null +++ b/test/examples/proximal/ridge.jl @@ -0,0 +1,61 @@ +using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators + +# load data +dat = example_data("political_democracy") + +############################################################################ +### define models +############################################################################ + +observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +latent_vars = [:ind60, :dem60, :dem65] + +graph = @StenoGraph begin + ind60 → fixed(1) * x1 + x2 + x3 + dem60 → fixed(1) * y1 + y2 + y3 + y4 + dem65 → fixed(1) * y5 + y6 + y7 + y8 + + dem60 ← ind60 + dem65 ← dem60 + dem65 ← ind60 + + _(observed_vars) ↔ _(observed_vars) + _(latent_vars) ↔ _(latent_vars) + + y1 ↔ label(:cov_15) * y5 + y2 ↔ label(:cov_24) * y4 + label(:cov_26) * y6 + y3 ↔ label(:cov_37) * y7 + y4 ↔ label(:cov_48) * y8 + y6 ↔ label(:cov_68) * y8 +end + +partable = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) + +ram_mat = RAMMatrices(partable) + +model = Sem(specification = partable, data = dat, loss = SemML) + +fit = sem_fit(model) + +# use ridge from StructuralEquationModels +model_ridge = Sem( + specification = partable, + data = dat, + loss = (SemML, SemRidge), + α_ridge = 0.02, + which_ridge = 16:20, +) + +solution_ridge = sem_fit(model_ridge) + +# use ridge from ProximalSEM; SqrNormL2 uses λ/2 as penalty +λ = zeros(31); +λ[16:20] .= 0.04; + +model_prox = Sem(specification = partable, data = dat, loss = SemML) + +solution_prox = sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) + +@testset "ridge_solution" begin + @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-4) +end From c19c4a7cb625d1afc48e372c75d2db0f4ddb508a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 19 Dec 2024 16:34:10 -0800 Subject: [PATCH 28/71] optim/documentation.jl: rename to abstract.jl --- src/StructuralEquationModels.jl | 2 +- src/optimizer/{documentation.jl => abstract.jl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/optimizer/{documentation.jl => abstract.jl} (100%) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ca1ae61f0..9e0fc3669 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -60,7 +60,7 @@ include("loss/regularization/ridge.jl") include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer -include("optimizer/documentation.jl") +include("optimizer/abstract.jl") include("optimizer/Empty.jl") include("optimizer/optim.jl") # helper functions diff --git a/src/optimizer/documentation.jl b/src/optimizer/abstract.jl similarity index 100% rename from src/optimizer/documentation.jl rename to src/optimizer/abstract.jl From c5b48c73bea5caf6aa5d5bd369e385e0ddec3272 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 20 Dec 2024 13:20:28 -0800 Subject: [PATCH 29/71] ext: change folder layout --- ext/{optimizer => SEMNLOptExt}/NLopt.jl | 0 ext/{ => SEMNLOptExt}/SEMNLOptExt.jl | 2 +- ext/{optimizer => SEMProximalOptExt}/ProximalAlgorithms.jl | 0 ext/{ => SEMProximalOptExt}/SEMProximalOptExt.jl | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename ext/{optimizer => SEMNLOptExt}/NLopt.jl (100%) rename ext/{ => SEMNLOptExt}/SEMNLOptExt.jl (82%) rename ext/{optimizer => SEMProximalOptExt}/ProximalAlgorithms.jl (100%) rename ext/{ => SEMProximalOptExt}/SEMProximalOptExt.jl (85%) diff --git a/ext/optimizer/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl similarity index 100% rename from ext/optimizer/NLopt.jl rename to ext/SEMNLOptExt/NLopt.jl diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl similarity index 82% rename from ext/SEMNLOptExt.jl rename to ext/SEMNLOptExt/SEMNLOptExt.jl index a727b82f1..a159f6dc8 100644 --- a/ext/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -6,6 +6,6 @@ SEM = StructuralEquationModels export SemOptimizerNLopt, NLoptConstraint -include("optimizer/NLopt.jl") +include("NLopt.jl") end diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl similarity index 100% rename from ext/optimizer/ProximalAlgorithms.jl rename to ext/SEMProximalOptExt/ProximalAlgorithms.jl diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl similarity index 85% rename from ext/SEMProximalOptExt.jl rename to ext/SEMProximalOptExt/SEMProximalOptExt.jl index e81760acb..8f91e03b0 100644 --- a/ext/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -9,6 +9,6 @@ SEM = StructuralEquationModels #ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) -include("optimizer/ProximalAlgorithms.jl") +include("ProximalAlgorithms.jl") end From d5357f0240543d44272150604e6f272da914802c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 24 Dec 2024 11:05:25 -0800 Subject: [PATCH 30/71] Project.toml: fix ProximalOperators ID --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1bd335f19..ed5239c94 100644 --- a/Project.toml +++ b/Project.toml @@ -48,7 +48,7 @@ test = ["Test"] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" -ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" +ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" [extensions] SEMNLOptExt = "NLopt" From 48a744f8976e3d055aaa7b29529a96e518597dc8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 31 Jul 2024 20:55:35 -0700 Subject: [PATCH 31/71] docs: fix nsamples, nobserved_vars --- docs/src/developer/observed.md | 8 ++++---- docs/src/tutorials/inspection/inspection.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/developer/observed.md b/docs/src/developer/observed.md index 2b695e597..93eca6ed9 100644 --- a/docs/src/developer/observed.md +++ b/docs/src/developer/observed.md @@ -22,10 +22,10 @@ end To compute some fit indices, you need to provide methods for ```julia -# Number of observed datapoints -n_obs(observed::MyObserved) = ... -# Number of manifest variables -n_man(observed::MyObserved) = ... +# Number of samples (observations) in the dataset +nsamples(observed::MyObserved) = ... +# Number of observed variables +nobserved_vars(observed::MyObserved) = ... ``` As always, you can add additional methods for properties that imply types and loss function want to access, for example (from the `SemObservedCommon` implementation): diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index b2eefadb2..88caf5812 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -1,7 +1,7 @@ # Model inspection ```@setup colored -using StructuralEquationModels +using StructuralEquationModels observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] latent_vars = [:ind60, :dem60, :dem65] @@ -32,7 +32,7 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, + latent_vars = latent_vars, observed_vars = observed_vars) data = example_data("political_democracy") @@ -128,8 +128,8 @@ BIC χ² df minus2ll -n_man -n_obs +nobserved_vars +nsamples nparams p_value RMSEA From 5faf1160d94ee62e1b84729f58a17f7da9653565 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 11:25:21 -0700 Subject: [PATCH 32/71] cleanup data columns reordering define a single source_to_dest_perm() function --- src/observed/abstract.jl | 30 ++++++++++++++++++++++++++++++ src/observed/covariance.jl | 33 +++------------------------------ src/observed/data.jl | 17 +---------------- src/observed/missing.jl | 2 +- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 90de8b5a6..71e87466a 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -8,3 +8,33 @@ Rows are samples, columns are observed variables. [`nsamples`](@ref), [`observed_vars`](@ref). """ samples(observed::SemObserved) = observed.data + +############################################################################################ +### Additional functions +############################################################################################ + +# compute the permutation that subsets and reorders source elements +# to match the destination order. +# if multiple identical elements are present in the source, the last one is used. +# if one_to_one is true, checks that the source and destination have the same length. +function source_to_dest_perm( + src::AbstractVector, + dest::AbstractVector; + one_to_one::Bool = false, + entities::String = "elements", +) + if dest == src # exact match + return eachindex(dest) + else + one_to_one && + length(dest) != length(src) && + throw( + DimensionMismatch( + "The length of the new $entities order ($(length(dest))) " * + "does not match the number of $entities ($(length(src)))", + ), + ) + src_inds = Dict(el => i for (i, el) in enumerate(src)) + return [src_inds[el] for el in dest] + end +end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index b78f41833..860391e21 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -75,9 +75,9 @@ function SemObservedCovariance(; end if !isnothing(spec_colnames) - obs_cov = reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - isnothing(obs_mean) || - (obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames)) + obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) + obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] + isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), nsamples) @@ -99,30 +99,3 @@ samples(observed::SemObservedCovariance) = obs_cov(observed::SemObservedCovariance) = observed.obs_cov obs_mean(observed::SemObservedCovariance) = observed.obs_mean - -############################################################################################ -### Additional functions -############################################################################################ - -# reorder covariance matrices -------------------------------------------------------------- -function reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_cov - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_cov = obs_cov[new_position, new_position] - return obs_cov - end -end - -# reorder means ---------------------------------------------------------------------------- - -function reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_mean - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_mean = obs_mean[new_position] - return obs_mean - end -end diff --git a/src/observed/data.jl b/src/observed/data.jl index c9b50e597..ff68b450a 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -91,7 +91,7 @@ function SemObservedData(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end @@ -121,18 +121,3 @@ nobserved_vars(observed::SemObservedData) = observed.nobs_vars obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean - -############################################################################################ -### Additional functions -############################################################################################ - -# reorder data ----------------------------------------------------------------------------- -function reorder_data(data::AbstractArray, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return data - else - obs_positions = Dict(col => i for (i, col) in enumerate(obs_colnames)) - new_positions = [obs_positions[col] for col in spec_colnames] - return data[:, new_positions] - end -end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index b628a313b..1eafab8f8 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -123,7 +123,7 @@ function SemObservedMissing(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end From 30e0b240d234a5600206abe6a1417fa84c6503d6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 11:57:28 -0800 Subject: [PATCH 33/71] SemObservedCov: def as an alias of SemObservedData reduces code duplication; also annotate types of ctor args now samples(SemObsCov) returns nothing --- src/StructuralEquationModels.jl | 2 +- src/observed/covariance.jl | 53 ++++++++------------------- test/unit_tests/data_input_formats.jl | 6 +-- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 9e0fc3669..1caf1f5b4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -41,8 +41,8 @@ include("frontend/fit/summary.jl") include("frontend/pretty_printing.jl") # observed include("observed/abstract.jl") -include("observed/covariance.jl") include("observed/data.jl") +include("observed/covariance.jl") include("observed/missing.jl") include("observed/EM.jl") # constructor diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 860391e21..195b84050 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -1,3 +1,10 @@ +""" +Type alias for [`SemObservedData`](@ref) that has mean and covariance, but no actual data. + +For instances of `SemObservedCovariance` [`samples`](@ref) returns `nothing`. +""" +const SemObservedCovariance{B, C} = SemObservedData{Nothing, B, C} + """ For observed covariance matrices and means. @@ -39,27 +46,19 @@ use this if you are sure your covariance matrix is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -struct SemObservedCovariance{B, C} <: SemObserved - obs_cov::B - obs_mean::C - nobs_vars::Int - nsamples::Int -end - function SemObservedCovariance(; specification::Union{SemSpecification, Nothing} = nothing, - obs_cov, - obs_colnames = nothing, - spec_colnames = nothing, - obs_mean = nothing, - meanstructure = false, + obs_cov::AbstractMatrix, + obs_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + spec_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + obs_mean::Union{AbstractVector, Nothing} = nothing, + meanstructure::Bool = false, nsamples::Integer, kwargs..., ) - if !meanstructure & !isnothing(obs_mean) + if !meanstructure && !isnothing(obs_mean) throw(ArgumentError("observed means were passed, but `meanstructure = false`")) - - elseif meanstructure & isnothing(obs_mean) + elseif meanstructure && isnothing(obs_mean) throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) end @@ -67,11 +66,8 @@ function SemObservedCovariance(; spec_colnames = observed_vars(specification) end - if !isnothing(spec_colnames) & isnothing(obs_colnames) + if !isnothing(spec_colnames) && isnothing(obs_colnames) throw(ArgumentError("no `obs_colnames` were specified")) - - elseif !isnothing(spec_colnames) & !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end if !isnothing(spec_colnames) @@ -80,22 +76,5 @@ function SemObservedCovariance(; isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - return SemObservedCovariance(obs_cov, obs_mean, size(obs_cov, 1), nsamples) + return SemObservedData(nothing, obs_cov, obs_mean, size(obs_cov, 1), nsamples) end - -############################################################################################ -### Recommended methods -############################################################################################ - -nsamples(observed::SemObservedCovariance) = observed.nsamples -nobserved_vars(observed::SemObservedCovariance) = observed.nobs_vars - -samples(observed::SemObservedCovariance) = - error("$(typeof(observed)) does not store data samples") - -############################################################################################ -### additional methods -############################################################################################ - -obs_cov(observed::SemObservedCovariance) = observed.obs_cov -obs_mean(observed::SemObservedCovariance) = observed.obs_mean diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 3fc255b84..9ab0c0af0 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -240,7 +240,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed) + @test @inferred(samples(observed)) === nothing observed_nospec = SemObservedCovariance( specification = nothing, @@ -260,7 +260,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed_nospec) + @test @inferred(samples(observed_nospec)) === nothing observed_shuffle = SemObservedCovariance( specification = spec, @@ -281,7 +281,7 @@ end # SemObservedData approx_cov = true, ) - @test_throws ErrorException samples(observed_shuffle) + @test @inferred(samples(observed_shuffle)) === nothing # respect specification order @test @inferred(obs_cov(observed_shuffle)) ≈ obs_cov(observed) From 86c5e2d8ba8a194f051a3de235f4509cd88e37b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 8 May 2024 18:18:01 -0700 Subject: [PATCH 34/71] SemObserved: store observed_vars add observed_vars(data::SemObserved) --- src/observed/abstract.jl | 2 ++ src/observed/covariance.jl | 3 ++- src/observed/data.jl | 9 +++++---- src/observed/missing.jl | 10 +++++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 71e87466a..62e88681b 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -9,6 +9,8 @@ Rows are samples, columns are observed variables. """ samples(observed::SemObserved) = observed.data +observed_vars(observed::SemObserved) = observed.observed_vars + ############################################################################################ ### Additional functions ############################################################################################ diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 195b84050..195d55b4e 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -72,9 +72,10 @@ function SemObservedCovariance(; if !isnothing(spec_colnames) obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) + obs_colnames = obs_colnames[obs2spec_perm] obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - return SemObservedData(nothing, obs_cov, obs_mean, size(obs_cov, 1), nsamples) + return SemObservedData(nothing, Symbol.(obs_colnames), obs_cov, obs_mean, nsamples) end diff --git a/src/observed/data.jl b/src/observed/data.jl index ff68b450a..700155924 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -39,9 +39,9 @@ use this if you are sure your observed data is in the right format. """ struct SemObservedData{A, B, C} <: SemObserved data::A + observed_vars::Vector{Symbol} obs_cov::B obs_mean::C - nobs_vars::Int nsamples::Int end @@ -68,6 +68,7 @@ function SemObservedData(; if isnothing(obs_colnames) try data = data[:, spec_colnames] + obs_colnames = spec_colnames catch throw( ArgumentError( @@ -91,7 +92,8 @@ function SemObservedData(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] + obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] + data = data[:, obs_colnames] end end @@ -101,9 +103,9 @@ function SemObservedData(; return SemObservedData( data, + Symbol.(obs_colnames), compute_covariance ? Statistics.cov(data) : nothing, meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, - size(data, 2), size(data, 1), ) end @@ -113,7 +115,6 @@ end ############################################################################################ nsamples(observed::SemObservedData) = observed.nsamples -nobserved_vars(observed::SemObservedData) = observed.nobs_vars ############################################################################################ ### additional methods diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 1eafab8f8..76dd70cbb 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -55,7 +55,6 @@ use this if you are sure your observed data is in the right format. """ mutable struct SemObservedMissing{ A <: AbstractArray, - D <: Number, O <: Number, P <: Vector, P2 <: Vector, @@ -68,7 +67,7 @@ mutable struct SemObservedMissing{ S <: EmMVNModel, } <: SemObserved data::A - nobs_vars::D + observed_vars::Vector{Symbol} nsamples::O patterns::P # missing patterns patterns_not::P2 @@ -100,6 +99,7 @@ function SemObservedMissing(; if isnothing(obs_colnames) try data = data[:, spec_colnames] + obs_colnames = spec_colnames catch throw( ArgumentError( @@ -123,7 +123,8 @@ function SemObservedMissing(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] + obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] + data = data[:, obs_colnames] end end @@ -186,7 +187,7 @@ function SemObservedMissing(; return SemObservedMissing( data, - nobs_vars, + Symbol.(obs_colnames), nsamples, remember_cart, remember_cart_not, @@ -205,7 +206,6 @@ end ############################################################################################ nsamples(observed::SemObservedMissing) = observed.nsamples -nobserved_vars(observed::SemObservedMissing) = observed.nobs_vars ############################################################################################ ### Additional methods From ef1861e16f204bd39ac91e3f2361e1ecb8703fa9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 8 May 2024 18:18:01 -0700 Subject: [PATCH 35/71] nsamples(observed::SemObserved): unify --- src/observed/abstract.jl | 1 + src/observed/data.jl | 2 -- src/observed/missing.jl | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 62e88681b..816dd9e80 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -8,6 +8,7 @@ Rows are samples, columns are observed variables. [`nsamples`](@ref), [`observed_vars`](@ref). """ samples(observed::SemObserved) = observed.data +nsamples(observed::SemObserved) = observed.nsamples observed_vars(observed::SemObserved) = observed.observed_vars diff --git a/src/observed/data.jl b/src/observed/data.jl index 700155924..ce4ce4bce 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -114,8 +114,6 @@ end ### Recommended methods ############################################################################################ -nsamples(observed::SemObservedData) = observed.nsamples - ############################################################################################ ### additional methods ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 76dd70cbb..0f95037be 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -205,8 +205,6 @@ end ### Recommended methods ############################################################################################ -nsamples(observed::SemObservedMissing) = observed.nsamples - ############################################################################################ ### Additional methods ############################################################################################ From 1d573a3e66a5384c4b03a3f980ead66f0e930e28 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:21:53 -0800 Subject: [PATCH 36/71] FIML: simplify index generation --- src/loss/ML/FIML.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 20c81b831..d88288453 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -61,9 +61,11 @@ function SemFIML(; observed, specification, kwargs...) imp_inv = zeros(nobs_vars, nobs_vars) mult = similar.(inverses) - ∇ind = vec(CartesianIndices(Array{Float64}(undef, nobs_vars, nobs_vars))) - ∇ind = - [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] + # linear indicies of co-observed variable pairs for each pattern + Σ_linind = LinearIndices((nobs_vars, nobs_vars)) + ∇ind = map(patterns_not(observed)) do pat_vars + vec(Σ_linind[pat_vars, pat_vars]) + end return SemFIML( ExactHessian(), From b8256678665846716fcb501542eebaa35738eca7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:04:22 -0800 Subject: [PATCH 37/71] SemObservedMissing: refactor * use SemObsMissingPattern struct to simplify code * replace O(Nvars^2) common pattern detection with Dict{} * don't store row-wise, store sub-matrices of non-missing data instead * use StatsBase.mean_and_cov() --- src/StructuralEquationModels.jl | 1 + src/frontend/fit/fitmeasures/minus2ll.jl | 71 +++--------- src/loss/ML/FIML.jl | 72 ++++++------- src/observed/EM.jl | 34 +++--- src/observed/missing.jl | 131 ++++++----------------- src/observed/missing_pattern.jl | 45 ++++++++ test/unit_tests/data_input_formats.jl | 9 +- 7 files changed, 147 insertions(+), 216 deletions(-) create mode 100644 src/observed/missing_pattern.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 1caf1f5b4..a6677a4ed 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -43,6 +43,7 @@ include("frontend/pretty_printing.jl") include("observed/abstract.jl") include("observed/data.jl") include("observed/covariance.jl") +include("observed/missing_pattern.jl") include("observed/missing.jl") include("observed/EM.jl") # constructor diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 54a4ce12d..1cddee71d 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -31,74 +31,33 @@ minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, loss_ml::SemWLS) = # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, loss_ml::SemFIML) - F = minimum - F *= nsamples(observed) - F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars) + F = minimum * nsamples(observed) + F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) return F end # compute likelihood for missing data - H1 ------------------------------------------------- # -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ) function minus2ll(observed::SemObservedMissing) - if observed.em_model.fitted - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - nsamples(observed), - pattern_rows(observed), - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_nsamples, - observed.pattern_nobs_vars, - ) - else - em_mvn(observed) - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - nsamples(observed), - pattern_rows(observed), - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_nsamples, - observed.pattern_nobs_vars, - ) - end -end - -function minus2ll( - μ, - Σ, - N, - rows, - patterns, - obs_mean, - obs_cov, - pattern_nsamples, - pattern_nobs_vars, -) - F = 0.0 + # fit EM-based mean and cov if not yet fitted + # FIXME EM could be very computationally expensive + observed.em_model.fitted || em_mvn(observed) - for i in 1:length(rows) - nᵢ = pattern_nsamples[i] - # missing pattern - pattern = patterns[i] - # observed data - Sᵢ = obs_cov[i] + Σ = observed.em_model.Σ + μ = observed.em_model.μ + F = sum(observed.patterns) do pat # implied covariance/mean - Σᵢ = Σ[pattern, pattern] - ld = logdet(Σᵢ) - Σᵢ⁻¹ = inv(cholesky(Σᵢ)) - meandiffᵢ = obs_mean[i] - μ[pattern] + Σᵢ = Σ[pat.measured_mask, pat.measured_mask] + Σᵢ_chol = cholesky!(Σᵢ) + ld = logdet(Σᵢ_chol) + Σᵢ⁻¹ = LinearAlgebra.inv!(Σᵢ_chol) + meandiffᵢ = pat.measured_mean - μ[pat.measured_mask] - F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ) + F_one_pattern(meandiffᵢ, Σᵢ⁻¹, pat.measured_cov, ld, nsamples(pat)) end - F += sum(log(2π) * pattern_nsamples .* pattern_nobs_vars) - #F *= N + F += log(2π) * sum(pat -> nsamples(pat) * nmeasured_vars(pat), observed.patterns) return F end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index d88288453..bf020d561 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -47,23 +47,25 @@ end ### Constructors ############################################################################################ -function SemFIML(; observed, specification, kwargs...) - inverses = broadcast(x -> zeros(x, x), pattern_nobs_vars(observed)) +function SemFIML(; observed::SemObservedMissing, specification, kwargs...) + inverses = + [zeros(nmeasured_vars(pat), nmeasured_vars(pat)) for pat in observed.patterns] choleskys = Array{Cholesky{Float64, Array{Float64, 2}}, 1}(undef, length(inverses)) - n_patterns = size(pattern_rows(observed), 1) + n_patterns = length(observed.patterns) logdets = zeros(n_patterns) - imp_mean = zeros.(pattern_nobs_vars(observed)) - meandiff = zeros.(pattern_nobs_vars(observed)) + imp_mean = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] + meandiff = [zeros(nmeasured_vars(pat)) for pat in observed.patterns] nobs_vars = nobserved_vars(observed) imp_inv = zeros(nobs_vars, nobs_vars) mult = similar.(inverses) - # linear indicies of co-observed variable pairs for each pattern + # generate linear indicies of co-observed variable pairs for each pattern Σ_linind = LinearIndices((nobs_vars, nobs_vars)) - ∇ind = map(patterns_not(observed)) do pat_vars + ∇ind = map(observed.patterns) do pat + pat_vars = findall(pat.measured_mask) vec(Σ_linind[pat_vars, pat_vars]) end @@ -106,10 +108,10 @@ function evaluate!( prepare_SemFIML!(semfiml, model) scale = inv(nsamples(observed(model))) - obs_rows = pattern_rows(observed(model)) - isnothing(objective) || (objective = scale * F_FIML(obs_rows, semfiml, model, params)) + isnothing(objective) || + (objective = scale * F_FIML(observed(model), semfiml, model, params)) isnothing(gradient) || - (∇F_FIML!(gradient, obs_rows, semfiml, model); gradient .*= scale) + (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) return objective end @@ -133,16 +135,16 @@ function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) return F * N end -function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, model) +function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model) diff⨉inv = μ_diff' * Σ⁻¹ if N > one(N) JΣ[∇ind] .+= N * vec(Σ⁻¹ * (I - S * Σ⁻¹ - μ_diff * diff⨉inv)) - @. Jμ[pattern] += (N * 2 * diff⨉inv)' + @. Jμ[obs_mask] += (N * 2 * diff⨉inv)' else JΣ[∇ind] .+= vec(Σ⁻¹ * (I - μ_diff * diff⨉inv)) - @. Jμ[pattern] += (2 * diff⨉inv)' + @. Jμ[obs_mask] += (2 * diff⨉inv)' end end @@ -165,32 +167,32 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) mul!(G, ∇μ', Jμ, -1, 1) end -function F_FIML(rows, semfiml, model, params) +function F_FIML(observed::SemObservedMissing, semfiml, model, params) F = zero(eltype(params)) - for i in 1:size(rows, 1) + for (i, pat) in enumerate(observed.patterns) F += F_one_pattern( semfiml.meandiff[i], semfiml.inverses[i], - obs_cov(observed(model))[i], + pat.measured_cov, semfiml.logdets[i], - pattern_nsamples(observed(model))[i], + nsamples(pat), ) end return F end -function ∇F_FIML!(G, rows, semfiml, model) +function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) Jμ = zeros(nobserved_vars(model)) JΣ = zeros(nobserved_vars(model)^2) - for i in 1:size(rows, 1) + for (i, pat) in enumerate(observed.patterns) ∇F_one_pattern( semfiml.meandiff[i], semfiml.inverses[i], - obs_cov(observed(model))[i], - patterns(observed(model))[i], + pat.measured_cov, + pat.measured_mask, semfiml.∇ind[i], - pattern_nsamples(observed(model))[i], + nsamples(pat), Jμ, JΣ, model, @@ -204,29 +206,21 @@ function prepare_SemFIML!(semfiml, model) batch_cholesky!(semfiml, model) #batch_sym_inv_update!(semfiml, model) batch_inv!(semfiml, model) - for i in 1:size(pattern_nsamples(observed(model)), 1) - semfiml.meandiff[i] .= obs_mean(observed(model))[i] - semfiml.imp_mean[i] + for (i, pat) in enumerate(observed(model).patterns) + semfiml.meandiff[i] .= pat.measured_mean .- semfiml.imp_mean[i] end end -function copy_per_pattern!(inverses, source_inverses, means, source_means, patterns) - @views for i in 1:size(patterns, 1) - inverses[i] .= source_inverses[patterns[i], patterns[i]] - end - - @views for i in 1:size(patterns, 1) - means[i] .= source_means[patterns[i]] +function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) + Σ = imply(model).Σ + μ = imply(model).μ + data = observed(model) + @inbounds @views for (i, pat) in enumerate(data.patterns) + fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask] + fiml.imp_mean[i] .= μ[pat.measured_mask] end end -copy_per_pattern!(semfiml, model::M where {M <: AbstractSem}) = copy_per_pattern!( - semfiml.inverses, - imply(model).Σ, - semfiml.imp_mean, - imply(model).μ, - patterns(observed(model)), -) - function batch_cholesky!(semfiml, model) for i in 1:size(semfiml.inverses, 1) semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i])) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index ef5da317d..beac45ca8 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -37,9 +37,9 @@ function em_mvn( 𝔼xxᵀ_pre = zeros(nvars, nvars) ### precompute for full cases - if length(observed.patterns[1]) == nvars - for row in pattern_rows(observed)[1] - row = observed.data_rowwise[row] + fullpat = observed.patterns[1] + if nmissed_vars(fullpat) == 0 + for row in eachrow(fullpat.data) 𝔼x_pre += row 𝔼xxᵀ_pre += row * row' end @@ -97,21 +97,27 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx Σ = em_model.Σ # Compute the expected sufficient statistics - for i in 2:length(observed.pattern_nsamples) + for pat in observed.patterns + (nmissed_vars(pat) == 0) && continue # skip full cases # observed and unobserved vars - u = observed.patterns_not[i] - o = observed.patterns[i] + u = pat.miss_mask + o = pat.measured_mask # precompute for pattern - V = Σ[u, u] - Σ[u, o] * (Σ[o, o] \ Σ[o, u]) + Σoo = Σ[o, o] + Σuo = Σ[u, o] + μu = μ[u] + μo = μ[o] + + V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u]) # loop trough data - for row in pattern_rows(observed)[i] - m = μ[u] + Σ[u, o] * (Σ[o, o] \ (observed.data_rowwise[row] - μ[o])) + for rowdata in eachrow(pat.data) + m = μu + Σuo * (Σoo \ (rowdata - μo)) 𝔼xᵢ[u] = m - 𝔼xᵢ[o] = observed.data_rowwise[row] + 𝔼xᵢ[o] = rowdata 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]' 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]' @@ -153,10 +159,10 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == nobserved_vars(observed)) & - (observed.pattern_nsamples[1] > 1) - μ = copy(observed.obs_mean[1]) - Σ = copy(Symmetric(observed.obs_cov[1])) + fullpat = observed.patterns[1] + if (nmissed_vars(fullpat) == 0) && (nobserved_vars(fullpat) > 1) + μ = copy(fullpat.measured_mean) + Σ = copy(Symmetric(fullpat.measured_cov)) if !isposdef(Σ) Σ = Matrix(Diagonal(Σ)) end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 0f95037be..96b027ae6 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -9,6 +9,10 @@ mutable struct EmMVNModel{A, b, B} fitted::B end +# FIXME type unstable +obs_mean(em::EmMVNModel) = ifelse(em.fitted, em.μ, nothing) +obs_cov(em::EmMVNModel) = ifelse(em.fitted, em.Σ, nothing) + """ For observed data with missing values. @@ -31,16 +35,7 @@ For observed data with missing values. - `nobserved_vars(::SemObservedMissing)` -> number of manifest variables - `samples(::SemObservedMissing)` -> observed data -- `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted - -- `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns -- `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern -- `pattern_rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern -- `pattern_nsamples(::SemObservedMissing)` -> number of data points per pattern -- `pattern_nobs_vars(::SemObservedMissing)` -> number of non-missing observed variables per pattern -- `obs_mean(::SemObservedMissing)` -> observed mean per pattern -- `obs_cov(::SemObservedMissing)` -> observed covariance per pattern -- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via optimization maximization +- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization ## Implementation Subtype of `SemObserved` @@ -53,31 +48,17 @@ use this if you are sure your observed data is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -mutable struct SemObservedMissing{ - A <: AbstractArray, - O <: Number, - P <: Vector, - P2 <: Vector, - R <: Vector, - PD <: AbstractArray, - PO <: AbstractArray, - PVO <: AbstractArray, - A2 <: AbstractArray, - A3 <: AbstractArray, - S <: EmMVNModel, +struct SemObservedMissing{ + T <: Real, + S <: Real, + E <: EmMVNModel, } <: SemObserved - data::A + data::Matrix{Union{T, Missing}} observed_vars::Vector{Symbol} - nsamples::O - patterns::P # missing patterns - patterns_not::P2 - pattern_rows::R # coresponding rows in data_rowwise - data_rowwise::PD # list of data - pattern_nsamples::PO # observed rows per pattern - pattern_nobs_vars::PVO # number of non-missing variables per pattern - obs_mean::A2 - obs_cov::A3 - em_model::S + nsamples::Int + patterns::Vector{SemObservedMissingPattern{T, S}} + + em_model::E end ############################################################################################ @@ -132,73 +113,27 @@ function SemObservedMissing(; data = Matrix(data) end - # remove persons with only missings - keep = Vector{Int64}() - for i in 1:size(data, 1) - if any(.!ismissing.(data[i, :])) - push!(keep, i) - end - end - data = data[keep, :] - nsamples, nobs_vars = size(data) - # compute and store the different missing patterns with their rowindices - missings = ismissing.(data) - patterns = [missings[i, :] for i in 1:size(missings, 1)] - - patterns_cart = findall.(!, patterns) - data_rowwise = [data[i, patterns_cart[i]] for i in 1:nsamples] - data_rowwise = convert.(Array{Float64}, data_rowwise) - - remember = Vector{BitArray{1}}() - rows = [Vector{Int64}(undef, 0) for i in 1:size(patterns, 1)] - for i in 1:size(patterns, 1) - unknown = true - for j in 1:size(remember, 1) - if patterns[i] == remember[j] - push!(rows[j], i) - unknown = false - end - end - if unknown - push!(remember, patterns[i]) - push!(rows[size(remember, 1)], i) + # detect all different missing patterns with their row indices + pattern_to_rows = Dict{BitVector, Vector{Int}}() + for (i, datarow) in zip(axes(data, 1), eachrow(data)) + pattern = BitVector(.!ismissing.(datarow)) + if sum(pattern) > 0 # skip all-missing rows + pattern_rows = get!(() -> Vector{Int}(), pattern_to_rows, pattern) + push!(pattern_rows, i) end end - rows = rows[1:length(remember)] - n_patterns = size(rows, 1) - - # sort by number of missings - sort_n_miss = sortperm(sum.(remember)) - remember = remember[sort_n_miss] - remember_cart = findall.(!, remember) - remember_cart_not = findall.(remember) - rows = rows[sort_n_miss] - - pattern_nsamples = size.(rows, 1) - pattern_nobs_vars = length.(remember_cart) - - cov_mean = [cov_and_mean(data_rowwise[rows]) for rows in rows] - obs_cov = [cov_mean[1] for cov_mean in cov_mean] - obs_mean = [cov_mean[2] for cov_mean in cov_mean] + # process each pattern and sort from most to least number of observed vars + patterns = [ + SemObservedMissingPattern(pat, rows, data) for (pat, rows) in pairs(pattern_to_rows) + ] + sort!(patterns, by = nmissed_vars) + # allocate EM model (but don't fit) em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) - return SemObservedMissing( - data, - Symbol.(obs_colnames), - nsamples, - remember_cart, - remember_cart_not, - rows, - data_rowwise, - pattern_nsamples, - pattern_nobs_vars, - obs_mean, - obs_cov, - em_model, - ) + return SemObservedMissing(data, Symbol.(obs_colnames), nsamples, patterns, em_model) end ############################################################################################ @@ -209,12 +144,6 @@ end ### Additional methods ############################################################################################ -patterns(observed::SemObservedMissing) = observed.patterns -patterns_not(observed::SemObservedMissing) = observed.patterns_not -pattern_rows(observed::SemObservedMissing) = observed.pattern_rows -data_rowwise(observed::SemObservedMissing) = observed.data_rowwise -pattern_nsamples(observed::SemObservedMissing) = observed.pattern_nsamples -pattern_nobs_vars(observed::SemObservedMissing) = observed.pattern_nobs_vars -obs_mean(observed::SemObservedMissing) = observed.obs_mean -obs_cov(observed::SemObservedMissing) = observed.obs_cov em_model(observed::SemObservedMissing) = observed.em_model +obs_mean(observed::SemObservedMissing) = obs_mean(em_model(observed)) +obs_cov(observed::SemObservedMissing) = obs_cov(em_model(observed)) diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl new file mode 100644 index 000000000..6ac6a360b --- /dev/null +++ b/src/observed/missing_pattern.jl @@ -0,0 +1,45 @@ +# data associated with the observed variables that all share the same missingness pattern +# variables that have values within that pattern are termed "measured" +# variables that have no measurements are termed "missing" +struct SemObservedMissingPattern{T, S} + measured_mask::BitVector # measured vars mask + miss_mask::BitVector # missing vars mask + rows::Vector{Int} # rows in original data + data::Matrix{T} # non-missing submatrix of data + + measured_mean::Vector{S} # means of measured vars + measured_cov::Matrix{S} # covariance of measured vars +end + +function SemObservedMissingPattern( + measured_mask::BitVector, + rows::AbstractVector{<:Integer}, + data::AbstractMatrix, +) + T = nonmissingtype(eltype(data)) + + pat_data = convert(Matrix{T}, view(data, rows, measured_mask)) + if size(pat_data, 1) > 1 + pat_mean, pat_cov = mean_and_cov(pat_data, 1, corrected = false) + @assert size(pat_cov) == (size(pat_data, 2), size(pat_data, 2)) + else + pat_mean = reshape(pat_data[1, :], 1, :) + # 1x1 covariance matrix since it is not meant to be used + pat_cov = fill(zero(T), 1, 1) + end + + return SemObservedMissingPattern{T, eltype(pat_mean)}( + measured_mask, + .!measured_mask, + rows, + pat_data, + dropdims(pat_mean, dims = 1), + pat_cov, + ) +end + +nobserved_vars(pat::SemObservedMissingPattern) = length(pat.measured_mask) +nsamples(pat::SemObservedMissingPattern) = length(pat.rows) + +nmeasured_vars(pat::SemObservedMissingPattern) = length(pat.measured_mean) +nmissed_vars(pat::SemObservedMissingPattern) = nobserved_vars(pat) - nmeasured_vars(pat) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 9ab0c0af0..8791ebc12 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -340,13 +340,10 @@ end # SemObservedCovariance meanstructure, ) - @test @inferred(length(StructuralEquationModels.patterns(observed))) == 55 - @test sum(@inferred(StructuralEquationModels.pattern_nsamples(observed))) == + @test @inferred(length(observed.patterns)) == 55 + @test sum(@inferred(nsamples(pat)) for pat in observed.patterns) == size(dat_missing, 1) - @test all( - <=(size(dat_missing, 2)), - @inferred(StructuralEquationModels.pattern_nsamples(observed)) - ) + @test all(nsamples(pat) <= size(dat_missing, 2) for pat in observed.patterns) observed_nospec = SemObservedMissing( specification = nothing, From a848ed36763fe99734d7c95c97a57ac07b289490 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 00:31:00 -0700 Subject: [PATCH 38/71] remove cov_and_mean(): not used anymore StatsBase.mean_and_cov() is used instead --- src/additional_functions/helper.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index be559b0d9..71b2559a8 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -98,11 +98,6 @@ function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind end end -function cov_and_mean(rows; corrected = false) - obs_mean, obs_cov = StatsBase.mean_and_cov(reduce(hcat, rows), 2, corrected = corrected) - return obs_cov, vec(obs_mean) -end - # n²×(n(n+1)/2) matrix to transform a vector of lower # triangular entries into a vectorized form of a n×n symmetric matrix, # opposite of elimination_matrix() From c952792f6f2aa7c3e27223e754f37287a25795cd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 11:57:28 -0800 Subject: [PATCH 39/71] SemObserved: unify data preparation - SemObservedData: parameterize by cov/mean eltype instead of the whole container types Co-authored-by: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> --- src/frontend/specification/Sem.jl | 18 ++++-- src/observed/abstract.jl | 102 ++++++++++++++++++++++++++++++ src/observed/covariance.jl | 96 ++++++++++++++-------------- src/observed/data.jl | 93 +++++---------------------- src/observed/missing.jl | 83 ++++++------------------ 5 files changed, 194 insertions(+), 198 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 741d5f3c6..28984dbe9 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -3,6 +3,7 @@ ############################################################################################ function Sem(; + specification = ParameterTable, observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, @@ -12,7 +13,7 @@ function Sem(; set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss = get_fields!(kwdict, observed, imply, loss) + observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) sem = Sem(observed, imply, loss) @@ -59,6 +60,7 @@ Returns the loss part of a model. loss(model::AbstractSemSingle) = model.loss function SemFiniteDiff(; + specification = ParameterTable, observed::O = SemObservedData, imply::I = RAM, loss::L = SemML, @@ -68,7 +70,7 @@ function SemFiniteDiff(; set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) - observed, imply, loss = get_fields!(kwdict, observed, imply, loss) + observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) sem = SemFiniteDiff(observed, imply, loss) @@ -96,23 +98,27 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) end # construct Sem fields -function get_fields!(kwargs, observed, imply, loss) +function get_fields!(kwargs, specification, observed, imply, loss) + if !isa(specification, SemSpecification) + specification = specification(; kwargs...) + end + # observed if !isa(observed, SemObserved) - observed = observed(; kwargs...) + observed = observed(; specification, kwargs...) end kwargs[:observed] = observed # imply if !isa(imply, SemImply) - imply = imply(; kwargs...) + imply = imply(; specification, kwargs...) end kwargs[:imply] = imply kwargs[:nparams] = nparams(imply) # loss - loss = get_SemLoss(loss; kwargs...) + loss = get_SemLoss(loss; specification, kwargs...) kwargs[:loss] = loss return observed, imply, loss diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 816dd9e80..53c0849c5 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -41,3 +41,105 @@ function source_to_dest_perm( return [src_inds[el] for el in dest] end end + +# function to prepare input data shared by SemObserved implementations +# returns tuple of +# 1) the matrix of data +# 2) the observed variable symbols that match matrix columns +# 3) the permutation of the original observed_vars (nothing if no reordering) +# If observed_vars is not specified, the vars order is taken from the specification. +# If both observed_vars and specification are provided, the observed_vars are used to match +# the column of the user-provided data matrix, and observed_vars(specification) is used to +# reorder the columns of the data to match the speciation. +# If no variable names are provided at all, generates the symbols in the form +# Symbol(observed_var_prefix, i) for i=1:nobserved_vars. +function prepare_data( + data::Union{AbstractDataFrame, AbstractMatrix, NTuple{2, Integer}, Nothing}, + observed_vars::Union{AbstractVector, Nothing}, + spec::Union{SemSpecification, Nothing}, +) + obs_vars = nothing + obs_vars_perm = nothing + if !isnothing(observed_vars) + obs_vars = Symbol.(observed_vars) + if !isnothing(spec) + obs_vars_spec = SEM.observed_vars(spec) + try + obs_vars_perm = source_to_dest_perm( + obs_vars, + obs_vars_spec, + one_to_one = false, + entities = "observed_vars", + ) + catch err + if isa(err, KeyError) + throw( + ArgumentError( + "observed_var \"$(err.key)\" from SEM specification is not listed in observed_vars argument", + ), + ) + else + rethrow(err) + end + end + # ignore trivial reorder + if obs_vars_perm == eachindex(obs_vars) + obs_vars_perm = nothing + end + end + elseif !isnothing(spec) + obs_vars = SEM.observed_vars(spec) + end + # observed vars in the order that matches the specification + obs_vars_reordered = isnothing(obs_vars_perm) ? obs_vars : obs_vars[obs_vars_perm] + + # subset the data, check that obs_vars matches data or guess the obs_vars + if data isa AbstractDataFrame + if !isnothing(obs_vars_reordered) # subset/reorder columns + data = data[:, obs_vars_reordered] + else # default symbol names + obs_vars = obs_vars_reordered = Symbol.(names(data)) + end + data_mtx = Matrix(data) + elseif data isa AbstractMatrix + if !isnothing(obs_vars) + size(data, 2) == length(obs_vars) || DimensionMismatch( + "The number of columns in the data matrix ($(size(data, 2))) does not match the length of observed_vars ($(length(obs_vars))).", + ) + # reorder columns to match the spec + data_ordered = !isnothing(obs_vars_perm) ? data[:, obs_vars_perm] : data + else + obs_vars = + obs_vars_reordered = + [Symbol(i) for i in axes(data, 2)] + data_ordered = data + end + # make sure data_mtx is a dense matrix (required for methods like mean_and_cov()) + data_mtx = convert(Matrix, data_ordered) + elseif data isa NTuple{2, Integer} # given the dimensions of the data matrix, but no data itself + data_mtx = nothing + nobs_vars = data[2] + if isnothing(obs_vars) + obs_vars = + obs_vars_reordered = [Symbol(i) for i in 1:nobs_vars] + elseif length(obs_vars) != nobs_vars + throw( + DimensionMismatch( + "The length of observed_vars ($(length(obs_vars))) does not match the data matrix columns ($(nobs_vars)).", + ), + ) + end + elseif isnothing(data) + data_mtx = nothing + if isnothing(obs_vars) + throw( + ArgumentError( + "No data, specification or observed_vars provided. Cannot infer observed_vars from provided inputs", + ), + ) + end + else + throw(ArgumentError("Unsupported data type: $(typeof(data))")) + end + return data_mtx, obs_vars_reordered, obs_vars_perm +end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 195d55b4e..08917116f 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -3,79 +3,77 @@ Type alias for [`SemObservedData`](@ref) that has mean and covariance, but no ac For instances of `SemObservedCovariance` [`samples`](@ref) returns `nothing`. """ -const SemObservedCovariance{B, C} = SemObservedData{Nothing, B, C} +const SemObservedCovariance{S} = SemObservedData{Nothing, S} """ -For observed covariance matrices and means. - -# Constructor - SemObservedCovariance(; specification, obs_cov, obs_colnames = nothing, meanstructure = false, obs_mean = nothing, - nsamples = nothing, + nsamples::Integer, kwargs...) -# Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) -- `obs_cov`: observed covariance matrix -- `obs_colnames::Vector{Symbol}`: column names of the covariance matrix -- `meanstructure::Bool`: does the model have a meanstructure? -- `obs_mean`: observed mean vector -- `nsamples::Number`: number of samples (observed data points); necessary for fit statistics - -# Extended help -## Interfaces -- `nsamples(::SemObservedCovariance)`: number of samples (observed data points) -- `n_man(::SemObservedCovariance)` -> number of manifest variables - -- `obs_cov(::SemObservedCovariance)` -> observed covariance matrix -- `obs_mean(::SemObservedCovariance)` -> observed means - -## Implementation -Subtype of `SemObserved` +Construct [`SemObserved`](@ref) without providing the observations data, +but with the covariations (`obs_cov`) and the means (`obs_means`) of the observed variables. -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your covariance matrix is in the right format. +Returns [`SemObservedCovariance`](@ref) object. -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object +# Arguments +- `obs_cov`: pre-computed covariations of the observed variables +- `obs_mean`: optional pre-computed means of the observed variables +- `observed_vars::AbstractVector`: IDs of the observed variables (rows and columns of the `obs_cov` matrix) +- `specification`: optional SEM specification ([`SemSpecification`](@ref)) +- `nsamples::Number`: number of samples (observed data points) used to compute `obs_cov` and `obs_means` + necessary for calculating fit statistics """ function SemObservedCovariance(; - specification::Union{SemSpecification, Nothing} = nothing, obs_cov::AbstractMatrix, - obs_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, - spec_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, obs_mean::Union{AbstractVector, Nothing} = nothing, - meanstructure::Bool = false, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, nsamples::Integer, kwargs..., ) - if !meanstructure && !isnothing(obs_mean) - throw(ArgumentError("observed means were passed, but `meanstructure = false`")) - elseif meanstructure && isnothing(obs_mean) - throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) - end + nvars = size(obs_cov, 1) + size(obs_cov, 2) == nvars || throw( + DimensionMismatch( + "The covariance matrix should be square, $(size(obs_cov)) was found.", + ), + ) + S = eltype(obs_cov) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) + if isnothing(obs_mean) + obs_mean = zeros(S, nvars) + else + length(obs_mean) == nvars || throw( + DimensionMismatch( + "The length of the mean vector $(length(obs_mean)) does not match the size of the covariance matrix $(size(obs_cov))", + ), + ) + S = promote_type(S, eltype(obs_mean)) end - if !isnothing(spec_colnames) && isnothing(obs_colnames) - throw(ArgumentError("no `obs_colnames` were specified")) + obs_cov = convert(Matrix{S}, obs_cov) + obs_mean = convert(Vector{S}, obs_mean) + + if !isnothing(observed_vars) + length(observed_vars) == nvars || throw( + DimensionMismatch( + "The length of the observed_vars $(length(observed_vars)) does not match the size of the covariance matrix $(size(obs_cov))", + ), + ) end - if !isnothing(spec_colnames) - obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) - obs_colnames = obs_colnames[obs2spec_perm] - obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] - isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) + _, obs_vars, obs_vars_perm = + prepare_data((nsamples, nvars), observed_vars, specification) + + # reorder to match the specification + if !isnothing(obs_vars_perm) + obs_cov = obs_cov[obs_vars_perm, obs_vars_perm] + obs_mean = obs_mean[obs_vars_perm] end - return SemObservedData(nothing, Symbol.(obs_colnames), obs_cov, obs_mean, nsamples) + return SemObservedData(nothing, obs_vars, obs_cov, obs_mean, nsamples) end diff --git a/src/observed/data.jl b/src/observed/data.jl index ce4ce4bce..4af00e5a3 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -4,17 +4,15 @@ For observed data without missings. # Constructor SemObservedData(; - specification, data, - meanstructure = false, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) # Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) -- `data`: observed data -- `meanstructure::Bool`: does the model have a meanstructure? -- `obs_colnames::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `specification`: optional SEM specification ([`SemSpecification`](@ref)) +- `data`: observed data -- *DataFrame* or *Matrix* +- `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) # Extended help ## Interfaces @@ -27,87 +25,26 @@ For observed data without missings. ## Implementation Subtype of `SemObserved` - -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your observed data is in the right format. - -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object -- `compute_covariance::Bool ) = true`: should the covariance of `data` be computed and stored? """ -struct SemObservedData{A, B, C} <: SemObserved - data::A +struct SemObservedData{D <: Union{Nothing, AbstractMatrix}, S <: Number} <: SemObserved + data::D observed_vars::Vector{Symbol} - obs_cov::B - obs_mean::C + obs_cov::Matrix{S} + obs_mean::Vector{S} nsamples::Int end -# error checks -function check_arguments_SemObservedData(kwargs...) - # data is a data frame, - -end - function SemObservedData(; - specification::Union{SemSpecification, Nothing}, data, - obs_colnames = nothing, - spec_colnames = nothing, - meanstructure = false, - compute_covariance = true, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, kwargs..., ) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) - end + data, obs_vars, _ = + prepare_data(data, observed_vars, specification) + obs_mean, obs_cov = mean_and_cov(data, 1) - if !isnothing(spec_colnames) - if isnothing(obs_colnames) - try - data = data[:, spec_colnames] - obs_colnames = spec_colnames - catch - throw( - ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ), - ) - end - else - if data isa DataFrame - throw( - ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ), - ) - end - - if !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) - end - - obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] - data = data[:, obs_colnames] - end - end - - if data isa DataFrame - data = Matrix(data) - end - - return SemObservedData( - data, - Symbol.(obs_colnames), - compute_covariance ? Statistics.cov(data) : nothing, - meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, - size(data, 1), - ) + return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) end ############################################################################################ diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 96b027ae6..de1c93c95 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -19,40 +19,28 @@ For observed data with missing values. # Constructor SemObservedMissing(; - specification, data, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) # Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) +- `specification`: optional SEM model specification ([`SemSpecification`](@ref)) - `data`: observed data -- `obs_colnames::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) +- `observed_vars::Vector{Symbol}`: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) # Extended help ## Interfaces -- `nsamples(::SemObservedMissing)` -> number of observed data points -- `nobserved_vars(::SemObservedMissing)` -> number of manifest variables +- `nsamples(::SemObservedMissing)` -> number of samples (data points) +- `nobserved_vars(::SemObservedMissing)` -> number of observed variables -- `samples(::SemObservedMissing)` -> observed data +- `samples(::SemObservedMissing)` -> data matrix (contains both measured and missing values) - `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization ## Implementation Subtype of `SemObserved` - -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your observed data is in the right format. - -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -struct SemObservedMissing{ - T <: Real, - S <: Real, - E <: EmMVNModel, -} <: SemObserved +struct SemObservedMissing{T <: Real, S <: Real, E <: EmMVNModel} <: SemObserved data::Matrix{Union{T, Missing}} observed_vars::Vector{Symbol} nsamples::Int @@ -66,53 +54,12 @@ end ############################################################################################ function SemObservedMissing(; - specification::Union{SemSpecification, Nothing}, data, - obs_colnames = nothing, - spec_colnames = nothing, + observed_vars::Union{AbstractVector, Nothing} = nothing, + specification::Union{SemSpecification, Nothing} = nothing, kwargs..., ) - if isnothing(spec_colnames) && !isnothing(specification) - spec_colnames = observed_vars(specification) - end - - if !isnothing(spec_colnames) - if isnothing(obs_colnames) - try - data = data[:, spec_colnames] - obs_colnames = spec_colnames - catch - throw( - ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ), - ) - end - else - if data isa DataFrame - throw( - ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ), - ) - end - - if !(eltype(obs_colnames) <: Symbol) - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) - end - - obs_colnames = obs_colnames[source_to_dest_perm(obs_colnames, spec_colnames)] - data = data[:, obs_colnames] - end - end - - if data isa DataFrame - data = Matrix(data) - end - + data, obs_vars, _ = prepare_data(data, observed_vars, specification) nsamples, nobs_vars = size(data) # detect all different missing patterns with their row indices @@ -133,7 +80,13 @@ function SemObservedMissing(; # allocate EM model (but don't fit) em_model = EmMVNModel(zeros(nobs_vars, nobs_vars), zeros(nobs_vars), false) - return SemObservedMissing(data, Symbol.(obs_colnames), nsamples, patterns, em_model) + return SemObservedMissing( + convert(Matrix{Union{nonmissingtype(eltype(data)), Missing}}, data), + obs_vars, + nsamples, + patterns, + em_model, + ) end ############################################################################################ From 2596c61ff889e0f4fd1a29cd7cc65d58694130d1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 28 Dec 2024 12:18:27 -0800 Subject: [PATCH 40/71] tests: update SemObserved tests to match the update data preparation behaviour --- test/unit_tests/data_input_formats.jl | 349 ++++++++++++++++---------- 1 file changed, 213 insertions(+), 136 deletions(-) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 8791ebc12..d93d02ad6 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -7,11 +7,19 @@ spec = ParameterTable( latent_vars = [:ind60, :dem60, :dem65], ) +# specification with non-existent observed var z1 +wrong_spec = ParameterTable( + observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :z1], + latent_vars = [:ind60, :dem60, :dem65], +) + ### data ----------------------------------------------------------------------------------- dat = example_data("political_democracy") dat_missing = example_data("political_democracy_missing")[:, names(dat)] +@assert Symbol.(names(dat)) == observed_vars(spec) + dat_matrix = Matrix(dat) dat_missing_matrix = Matrix(dat_missing) @@ -21,7 +29,12 @@ dat_mean = vcat(Statistics.mean(dat_matrix, dims = 1)...) # shuffle variables new_order = [3, 2, 7, 8, 5, 6, 9, 11, 1, 10, 4] -shuffle_names = Symbol.(names(dat))[new_order] +shuffle_names = names(dat)[new_order] + +shuffle_spec = ParameterTable( + observed_vars = Symbol.(shuffle_names), + latent_vars = [:ind60, :dem60, :dem65], +) shuffle_dat = dat[:, new_order] shuffle_dat_missing = dat_missing[:, new_order] @@ -29,8 +42,8 @@ shuffle_dat_missing = dat_missing[:, new_order] shuffle_dat_matrix = dat_matrix[:, new_order] shuffle_dat_missing_matrix = dat_missing_matrix[:, new_order] -shuffle_dat_cov = Statistics.cov(shuffle_dat_matrix) -shuffle_dat_mean = vcat(Statistics.mean(shuffle_dat_matrix, dims = 1)...) +shuffle_dat_cov = cov(shuffle_dat_matrix) +shuffle_dat_mean = vec(mean(shuffle_dat_matrix, dims = 1)) # common tests for SemObserved subtypes function test_observed( @@ -42,17 +55,16 @@ function test_observed( meanstructure::Bool, approx_cov::Bool = false, ) - @test @inferred(nobserved_vars(observed)) == size(dat, 2) - # FIXME observed should provide names of observed variables - @test @inferred(observed_vars(observed)) == names(dat) broken = true - @test @inferred(nsamples(observed)) == size(dat, 1) - - hasmissing = - !isnothing(dat_matrix) && any(ismissing, dat_matrix) || - !isnothing(dat_cov) && any(ismissing, dat_cov) + if !isnothing(dat) + @test @inferred(nsamples(observed)) == size(dat, 1) + @test @inferred(nobserved_vars(observed)) == size(dat, 2) + @test @inferred(observed_vars(observed)) == Symbol.(names(dat)) + end if !isnothing(dat_matrix) - if hasmissing + @test @inferred(nsamples(observed)) == size(dat_matrix, 1) + + if any(ismissing, dat_matrix) @test isequal(@inferred(samples(observed)), dat_matrix) else @test @inferred(samples(observed)) == dat_matrix @@ -60,7 +72,7 @@ function test_observed( end if !isnothing(dat_cov) - if hasmissing + if any(ismissing, dat_cov) @test isequal(@inferred(obs_cov(observed)), dat_cov) else if approx_cov @@ -72,17 +84,17 @@ function test_observed( end # FIXME actually, SemObserved should not use meanstructure and always provide obs_mean() - # meanstructure is a part of SEM model + # since meanstructure belongs to the implied part of a SEM model if meanstructure if !isnothing(dat_mean) - if hasmissing + if any(ismissing, dat_mean) @test isequal(@inferred(obs_mean(observed)), dat_mean) else @test @inferred(obs_mean(observed)) == dat_mean end else - # FIXME if meanstructure is present, obs_mean() should provide something (currently Missing don't support it) - @test (@inferred(obs_mean(observed)) isa AbstractVector{Float64}) broken = true + # FIXME @inferred is broken for EM cov/mean since it may return nothing if EM was not run + @test @inferred(obs_mean(observed)) isa AbstractVector{Float64} broken = true # EM-based means end else @test @inferred(obs_mean(observed)) === nothing skip = true @@ -93,32 +105,25 @@ end @testset "SemObservedData" begin # errors - @test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ) begin - SemObservedData( - specification = spec, - data = dat, - obs_colnames = Symbol.(names(dat)), - ) - end + obs_data_redundant = SemObservedData( + specification = spec, + data = dat, + observed_vars = Symbol.(names(dat)), + ) + @test observed_vars(obs_data_redundant) == Symbol.(names(dat)) + @test observed_vars(obs_data_redundant) == observed_vars(spec) - @test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ) begin - SemObservedData(specification = spec, data = dat_matrix) - end + obs_data_spec = SemObservedData(specification = spec, data = dat_matrix) + @test observed_vars(obs_data_spec) == observed_vars(spec) - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedData(specification = spec, data = dat_matrix, obs_colnames = names(dat)) - end + obs_data_strnames = + SemObservedData(specification = spec, data = dat_matrix, observed_vars = names(dat)) + @test observed_vars(obs_data_strnames) == Symbol.(names(dat)) @test_throws UndefKeywordError(:data) SemObservedData(specification = spec) - @test_throws UndefKeywordError(:specification) SemObservedData(data = dat_matrix) + obs_data_nonames = SemObservedData(data = dat_matrix) + @test observed_vars(obs_data_nonames) == Symbol.(1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = SemObservedData(specification = spec, data = dat; meanstructure) @@ -128,35 +133,92 @@ end observed_nospec = SemObservedData(specification = nothing, data = dat_matrix; meanstructure) - test_observed(observed_nospec, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + test_observed( + observed_nospec, + nothing, + dat_matrix, + dat_cov, + dat_mean; + meanstructure, + ) observed_matrix = SemObservedData( specification = spec, data = dat_matrix, - obs_colnames = Symbol.(names(dat)), - meanstructure = meanstructure, + observed_vars = Symbol.(names(dat)); + meanstructure, ) test_observed(observed_matrix, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + # detect non-existing column + @test_throws "ArgumentError: column name \"z1\"" SemObservedData( + specification = wrong_spec, + data = shuffle_dat, + ) + + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedData( + specification = wrong_spec, + data = shuffle_dat_matrix, + observed_vars = shuffle_names, + ) + + # cannot infer observed_vars + @test_throws "No data, specification or observed_vars provided" SemObservedData( + data = nothing, + ) + + if false # FIXME data = nothing is for simulation studies + # no data, just observed_vars + observed_nodata = + SemObservedData(data = nothing, observed_vars = Symbol.(names(dat))) + @test observed_nodata isa SemObservedData + @test @inferred(samples(observed_nodata)) === nothing + @test observed_vars(observed_nodata) == Symbol.(names(dat)) + end + + # spec takes precedence in obs_vars order + observed_spec = SemObservedData( + specification = spec, + data = shuffle_dat, + observed_vars = shuffle_names, + ) + + test_observed( + observed_spec, + dat, + dat_matrix, + dat_cov, + meanstructure ? dat_mean : nothing; + meanstructure, + ) + observed_shuffle = - SemObservedData(specification = spec, data = shuffle_dat; meanstructure) + SemObservedData(specification = shuffle_spec, data = shuffle_dat; meanstructure) - test_observed(observed_shuffle, dat, dat_matrix, dat_cov, dat_mean; meanstructure) + test_observed( + observed_shuffle, + shuffle_dat, + shuffle_dat_matrix, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; + meanstructure, + ) observed_matrix_shuffle = SemObservedData( - specification = spec, + specification = shuffle_spec, data = shuffle_dat_matrix, - obs_colnames = shuffle_names; + observed_vars = shuffle_names; meanstructure, ) test_observed( observed_matrix_shuffle, - dat, - dat_matrix, - dat_cov, - dat_mean; + shuffle_dat, + shuffle_dat_matrix, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; meanstructure, ) end # meanstructure @@ -170,43 +232,6 @@ end # SemObservedData @test_throws UndefKeywordError(:nsamples) SemObservedCovariance(obs_cov = dat_cov) - @test_throws ArgumentError("no `obs_colnames` were specified") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - nsamples = size(dat, 1), - ) - end - - @test_throws ArgumentError("observed means were passed, but `meanstructure = false`") begin - SemObservedCovariance( - specification = nothing, - obs_cov = dat_cov, - obs_mean = dat_mean, - nsamples = size(dat, 1), - ) - end - - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = names(dat), - nsamples = size(dat, 1), - meanstructure = false, - ) - end - - @test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin - SemObservedCovariance( - specification = spec, - obs_cov = dat_cov, - obs_colnames = Symbol.(names(dat)), - meanstructure = true, - nsamples = size(dat, 1), - ) - end - @testset "meanstructure=$meanstructure" for meanstructure in (false, true) # errors @@ -220,12 +245,25 @@ end # SemObservedData meanstructure, ) - # should work + # default vars + observed_nonames = SemObservedCovariance( + obs_cov = dat_cov, + obs_mean = meanstructure ? dat_mean : nothing, + nsamples = size(dat, 1), + ) + @test observed_vars(observed_nonames) == Symbol.("obs", 1:size(dat_cov, 2)) + + @test_throws DimensionMismatch SemObservedCovariance( + obs_cov = dat_cov, + observed_vars = Symbol.("obs", 1:(size(dat_cov, 2)+1)), + nsamples = size(dat, 1), + ) + observed = SemObservedCovariance( specification = spec, obs_cov = dat_cov, obs_mean = meanstructure ? dat_mean : nothing, - obs_colnames = obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), nsamples = size(dat, 1), meanstructure = meanstructure, ) @@ -252,7 +290,7 @@ end # SemObservedData test_observed( observed_nospec, - dat, + nothing, nothing, dat_cov, dat_mean; @@ -262,30 +300,51 @@ end # SemObservedData @test @inferred(samples(observed_nospec)) === nothing - observed_shuffle = SemObservedCovariance( + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedCovariance( + specification = wrong_spec, + obs_cov = shuffle_dat_cov, + observed_vars = shuffle_names, + nsamples = size(dat, 1), + ) + + # spec takes precedence in obs_vars order + observed_spec = SemObservedCovariance( specification = spec, obs_cov = shuffle_dat_cov, - obs_mean = meanstructure ? dat_mean[new_order] : nothing, - obs_colnames = shuffle_names, - nsamples = size(dat, 1); - meanstructure, + obs_mean = meanstructure ? shuffle_dat_mean : nothing, + observed_vars = shuffle_names, + nsamples = size(dat, 1), ) test_observed( - observed_shuffle, + observed_spec, dat, nothing, dat_cov, - dat_mean; + meanstructure ? dat_mean : nothing; meanstructure, approx_cov = true, ) - @test @inferred(samples(observed_shuffle)) === nothing + observed_shuffle = SemObservedCovariance( + specification = shuffle_spec, + obs_cov = shuffle_dat_cov, + obs_mean = meanstructure ? shuffle_dat_mean : nothing, + observed_vars = shuffle_names, + nsamples = size(dat, 1); + meanstructure, + ) - # respect specification order - @test @inferred(obs_cov(observed_shuffle)) ≈ obs_cov(observed) - @test @inferred(observed_vars(observed_shuffle)) == shuffle_names broken = true + test_observed( + observed_shuffle, + shuffle_dat, + nothing, + shuffle_dat_cov, + meanstructure ? shuffle_dat_mean : nothing; + meanstructure, + approx_cov = true, + ) end # meanstructure end # SemObservedCovariance @@ -294,38 +353,27 @@ end # SemObservedCovariance @testset "SemObservedMissing" begin # errors - @test_throws ArgumentError( - "You passed your data as a `DataFrame`, but also specified `obs_colnames`. " * - "Please make sure the column names of your data frame indicate the correct variables " * - "or pass your data in a different format.", - ) begin - SemObservedMissing( - specification = spec, - data = dat_missing, - obs_colnames = Symbol.(names(dat)), - ) - end + observed_redundant_names = SemObservedMissing( + specification = spec, + data = dat_missing, + observed_vars = Symbol.(names(dat)), + ) + @test observed_vars(observed_redundant_names) == Symbol.(names(dat)) - @test_throws ArgumentError( - "Your `data` can not be indexed by symbols. " * - "Maybe you forgot to provide column names via the `obs_colnames = ...` argument.", - ) begin - SemObservedMissing(specification = spec, data = dat_missing_matrix) - end + observed_spec_only = SemObservedMissing(specification = spec, data = dat_missing_matrix) + @test observed_vars(observed_spec_only) == observed_vars(spec) - @test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin - SemObservedMissing( - specification = spec, - data = dat_missing_matrix, - obs_colnames = names(dat), - ) - end + observed_str_colnames = SemObservedMissing( + specification = spec, + data = dat_missing_matrix, + observed_vars = names(dat), + ) + @test observed_vars(observed_str_colnames) == Symbol.(names(dat)) @test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) - @test_throws UndefKeywordError(:specification) SemObservedMissing( - data = dat_missing_matrix, - ) + observed_no_names = SemObservedMissing(data = dat_missing_matrix) + @test observed_vars(observed_no_names) == Symbol.(1:size(dat_missing_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = @@ -353,7 +401,7 @@ end # SemObservedCovariance test_observed( observed_nospec, - dat_missing, + nothing, dat_missing_matrix, nothing, nothing; @@ -363,7 +411,7 @@ end # SemObservedCovariance observed_matrix = SemObservedMissing( specification = spec, data = dat_missing_matrix, - obs_colnames = Symbol.(names(dat)), + observed_vars = Symbol.(names(dat)), ) test_observed( @@ -375,11 +423,28 @@ end # SemObservedCovariance meanstructure, ) - observed_shuffle = - SemObservedMissing(specification = spec, data = shuffle_dat_missing) + # detect non-existing column + @test_throws "ArgumentError: column name \"z1\"" SemObservedMissing( + specification = wrong_spec, + data = shuffle_dat, + ) + + # detect non-existing observed_var + @test_throws "ArgumentError: observed_var \"z1\"" SemObservedMissing( + specification = wrong_spec, + data = shuffle_dat_missing_matrix, + observed_vars = shuffle_names, + ) + + # spec takes precedence in obs_vars order + observed_spec = SemObservedMissing( + specification = spec, + observed_vars = shuffle_names, + data = shuffle_dat_missing, + ) test_observed( - observed_shuffle, + observed_spec, dat_missing, dat_missing_matrix, nothing, @@ -387,16 +452,28 @@ end # SemObservedCovariance meanstructure, ) + observed_shuffle = + SemObservedMissing(specification = shuffle_spec, data = shuffle_dat_missing) + + test_observed( + observed_shuffle, + shuffle_dat_missing, + shuffle_dat_missing_matrix, + nothing, + nothing; + meanstructure, + ) + observed_matrix_shuffle = SemObservedMissing( - specification = spec, + specification = shuffle_spec, data = shuffle_dat_missing_matrix, - obs_colnames = shuffle_names, + observed_vars = shuffle_names, ) test_observed( observed_matrix_shuffle, - dat_missing, - dat_missing_matrix, + shuffle_dat_missing, + shuffle_dat_missing_matrix, nothing, nothing; meanstructure, From 80a64c90667c119f3b513fdef70daf3513f18c97 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 1 Jan 2025 13:06:40 -0800 Subject: [PATCH 41/71] prep_data: warn if obs_vars order don't match spec --- src/observed/abstract.jl | 3 +++ test/unit_tests/data_input_formats.jl | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index 53c0849c5..fb31d9752 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -97,6 +97,9 @@ function prepare_data( if data isa AbstractDataFrame if !isnothing(obs_vars_reordered) # subset/reorder columns data = data[:, obs_vars_reordered] + if obs_vars_reordered != obs_vars + @warn "The order of variables in observed_vars argument does not match the order of observed_vars(specification). The specification order is used." + end else # default symbol names obs_vars = obs_vars_reordered = Symbol.(names(data)) end diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index d93d02ad6..fe9421f55 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -178,6 +178,12 @@ end @test observed_vars(observed_nodata) == Symbol.(names(dat)) end + @test_warn "The order of variables in observed_vars" SemObservedData( + specification = spec, + data = shuffle_dat, + observed_vars = shuffle_names, + ) + # spec takes precedence in obs_vars order observed_spec = SemObservedData( specification = spec, From 673aa2b9cf272a371e039f71b3789d685ce13cd8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 25 Dec 2024 13:11:47 -0800 Subject: [PATCH 42/71] SemObsData: observed_var_prefix kwarg to specify the prefix of the generated observed_vars if none provided could be inferred, defaults to :obs --- src/observed/abstract.jl | 11 ++++++++--- src/observed/covariance.jl | 3 ++- src/observed/data.jl | 3 ++- src/observed/missing.jl | 4 +++- test/unit_tests/data_input_formats.jl | 12 ++++++++++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/observed/abstract.jl b/src/observed/abstract.jl index fb31d9752..bb92ea12e 100644 --- a/src/observed/abstract.jl +++ b/src/observed/abstract.jl @@ -16,6 +16,10 @@ observed_vars(observed::SemObserved) = observed.observed_vars ### Additional functions ############################################################################################ +# generate default observed variable names if none provided +default_observed_vars(nobserved_vars::Integer, prefix::Union{Symbol, AbstractString}) = + Symbol.(prefix, 1:nobserved_vars) + # compute the permutation that subsets and reorders source elements # to match the destination order. # if multiple identical elements are present in the source, the last one is used. @@ -56,7 +60,8 @@ end function prepare_data( data::Union{AbstractDataFrame, AbstractMatrix, NTuple{2, Integer}, Nothing}, observed_vars::Union{AbstractVector, Nothing}, - spec::Union{SemSpecification, Nothing}, + spec::Union{SemSpecification, Nothing}; + observed_var_prefix::Union{Symbol, AbstractString}, ) obs_vars = nothing obs_vars_perm = nothing @@ -114,7 +119,7 @@ function prepare_data( else obs_vars = obs_vars_reordered = - [Symbol(i) for i in axes(data, 2)] + default_observed_vars(size(data, 2), observed_var_prefix) data_ordered = data end # make sure data_mtx is a dense matrix (required for methods like mean_and_cov()) @@ -124,7 +129,7 @@ function prepare_data( nobs_vars = data[2] if isnothing(obs_vars) obs_vars = - obs_vars_reordered = [Symbol(i) for i in 1:nobs_vars] + obs_vars_reordered = default_observed_vars(nobs_vars, observed_var_prefix) elseif length(obs_vars) != nobs_vars throw( DimensionMismatch( diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 08917116f..221ef5ca3 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -34,6 +34,7 @@ function SemObservedCovariance(; observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, nsamples::Integer, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) nvars = size(obs_cov, 1) @@ -67,7 +68,7 @@ function SemObservedCovariance(; end _, obs_vars, obs_vars_perm = - prepare_data((nsamples, nvars), observed_vars, specification) + prepare_data((nsamples, nvars), observed_vars, specification; observed_var_prefix) # reorder to match the specification if !isnothing(obs_vars_perm) diff --git a/src/observed/data.jl b/src/observed/data.jl index 4af00e5a3..b6ddaa43d 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -38,10 +38,11 @@ function SemObservedData(; data, observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) data, obs_vars, _ = - prepare_data(data, observed_vars, specification) + prepare_data(data, observed_vars, specification; observed_var_prefix) obs_mean, obs_cov = mean_and_cov(data, 1) return SemObservedData(data, obs_vars, obs_cov, vec(obs_mean), size(data, 1)) diff --git a/src/observed/missing.jl b/src/observed/missing.jl index de1c93c95..cf699252e 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -57,9 +57,11 @@ function SemObservedMissing(; data, observed_vars::Union{AbstractVector, Nothing} = nothing, specification::Union{SemSpecification, Nothing} = nothing, + observed_var_prefix::Union{Symbol, AbstractString} = :obs, kwargs..., ) - data, obs_vars, _ = prepare_data(data, observed_vars, specification) + data, obs_vars, _ = + prepare_data(data, observed_vars, specification; observed_var_prefix) nsamples, nobs_vars = size(data) # detect all different missing patterns with their row indices diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index fe9421f55..cc72673a6 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -123,7 +123,11 @@ end @test_throws UndefKeywordError(:data) SemObservedData(specification = spec) obs_data_nonames = SemObservedData(data = dat_matrix) - @test observed_vars(obs_data_nonames) == Symbol.(1:size(dat_matrix, 2)) + @test observed_vars(obs_data_nonames) == Symbol.("obs", 1:size(dat_matrix, 2)) + + obs_data_nonames2 = + SemObservedData(data = dat_matrix, observed_var_prefix = "observed_") + @test observed_vars(obs_data_nonames2) == Symbol.("observed_", 1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = SemObservedData(specification = spec, data = dat; meanstructure) @@ -379,7 +383,11 @@ end # SemObservedCovariance @test_throws UndefKeywordError(:data) SemObservedMissing(specification = spec) observed_no_names = SemObservedMissing(data = dat_missing_matrix) - @test observed_vars(observed_no_names) == Symbol.(1:size(dat_missing_matrix, 2)) + @test observed_vars(observed_no_names) == Symbol.(:obs, 1:size(dat_missing_matrix, 2)) + + observed_no_names2 = + SemObservedMissing(data = dat_missing_matrix, observed_var_prefix = "observed_") + @test observed_vars(observed_no_names2) == Symbol.("observed_", 1:size(dat_matrix, 2)) @testset "meanstructure=$meanstructure" for meanstructure in (false, true) observed = From 8c7cd1430fd0295a2d2eaf74f1ae098b63157174 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Thu, 2 Jan 2025 10:42:01 +0100 Subject: [PATCH 43/71] ParTable: add graph-based kw-only constructor --- src/frontend/specification/ParameterTable.jl | 4 ++-- src/frontend/specification/StenoGraphs.jl | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 8b7cc0973..07c24e46e 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -27,9 +27,9 @@ empty_partable_columns(nrows::Integer = 0) = Dict{Symbol, Vector}( :param => fill(Symbol(), nrows), ) -# construct using the provided columns data or create and empty table +# construct using the provided columns data or create an empty table function ParameterTable( - columns::Dict{Symbol, Vector} = empty_partable_columns(); + columns::Dict{Symbol, Vector}; observed_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, latent_vars::Union{AbstractVector{Symbol}, Nothing} = nothing, params::Union{AbstractVector{Symbol}, Nothing} = nothing, diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 5cf87c07a..65bace302 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -129,6 +129,17 @@ function ParameterTable( return ParameterTable(columns; latent_vars, observed_vars, params) end +############################################################################################ +### keyword only constructor (for call in `Sem` constructor) +############################################################################################ + +# FIXME: this kw-only ctor conflicts with the empty ParTable constructor; +# it is left here for compatibility with the current Sem construction API, +# the proper fix would be to move away from kw-only ctors in general +ParameterTable(; graph::Union{AbstractStenoGraph, Nothing} = nothing, kwargs...) = + !isnothing(graph) ? ParameterTable(graph; kwargs...) : + ParameterTable(empty_partable_columns(); kwargs...) + ############################################################################################ ### constructor for EnsembleParameterTable from graph ############################################################################################ From ff67cf71d00ab226bc6de86a3b6ef6d8ba190f21 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 6 Jan 2025 11:42:01 -0800 Subject: [PATCH 44/71] Project.toml: fix ProximalAlgorithms to 0.5 v0.7 changed the diff interface (v0.6 was skipped) --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index ed5239c94..b0e421f21 100644 --- a/Project.toml +++ b/Project.toml @@ -34,6 +34,7 @@ NLSolversBase = "7" NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" +ProximalAlgorithms = "0.5" StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6" SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" From e63d5d8cba8fbdf90309a65b2480d0f6fa6b67a8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 6 Jan 2025 12:57:22 -0800 Subject: [PATCH 45/71] switch to ProximalAlgorithms.jl v0.7 also drop ProximalOperators and ProximalCore weak deps --- Project.toml | 6 ++---- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 11 ++++++++--- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 4 +--- test/Project.toml | 1 - test/examples/proximal/l0.jl | 2 +- test/examples/proximal/lasso.jl | 2 +- test/examples/proximal/ridge.jl | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Project.toml b/Project.toml index b0e421f21..5937930d3 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ NLSolversBase = "7" NLopt = "0.6, 1" Optim = "1" PrettyTables = "2" -ProximalAlgorithms = "0.5" +ProximalAlgorithms = "0.7" StatsBase = "0.33, 0.34" Symbolics = "4, 5, 6" SymbolicUtils = "1.4 - 1.5, 1.7, 2, 3" @@ -48,9 +48,7 @@ test = ["Test"] [weakdeps] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" -ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" -ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" [extensions] SEMNLOptExt = "NLopt" -SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] +SEMProximalOptExt = "ProximalAlgorithms" diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 13debf79d..f82c2b005 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -54,9 +54,14 @@ function Base.show(io::IO, struct_inst::SemOptimizerProximal) print_field_types(io, struct_inst) end -## connect do ProximalAlgorithms.jl as backend -ProximalCore.gradient!(grad, model::AbstractSem, parameters) = - objective_gradient!(grad, model::AbstractSem, parameters) +## connect to ProximalAlgorithms.jl +function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) + grad = similar(params) + obj = SEM.evaluate!(zero(eltype(params)), grad, nothing, model, params) + return obj, grad +end + +#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) mutable struct ProximalResult result::Any diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 8f91e03b0..156311367 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -1,14 +1,12 @@ module SEMProximalOptExt using StructuralEquationModels -using ProximalCore, ProximalAlgorithms, ProximalOperators +using ProximalAlgorithms export SemOptimizerProximal SEM = StructuralEquationModels -#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) - include("ProximalAlgorithms.jl") end diff --git a/test/Project.toml b/test/Project.toml index 14bd0bece..59db0b155 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -10,7 +10,6 @@ NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" -ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" diff --git a/test/examples/proximal/l0.jl b/test/examples/proximal/l0.jl index e8874fd51..da20f3901 100644 --- a/test/examples/proximal/l0.jl +++ b/test/examples/proximal/l0.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") diff --git a/test/examples/proximal/lasso.jl b/test/examples/proximal/lasso.jl index 31a4073f9..314453df4 100644 --- a/test/examples/proximal/lasso.jl +++ b/test/examples/proximal/lasso.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 120910234..16a318a12 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalCore, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators # load data dat = example_data("political_democracy") From e2d6aa1b5baf0f119cb688313a1f24c86ef81492 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 11 Aug 2024 12:07:08 -0700 Subject: [PATCH 46/71] move params() to common.jl it is available for many SEM types, not just SemSpec --- src/frontend/common.jl | 7 +++++++ src/frontend/specification/documentation.jl | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/frontend/common.jl b/src/frontend/common.jl index 2be13c113..41d03effb 100644 --- a/src/frontend/common.jl +++ b/src/frontend/common.jl @@ -1,5 +1,12 @@ # API methods supported by multiple SEM.jl types +""" + params(semobj) -> Vector{Symbol} + +Return the vector of SEM model parameter identifiers. +""" +function params end + """ nparams(semobj) diff --git a/src/frontend/specification/documentation.jl b/src/frontend/specification/documentation.jl index 46135ead0..72d95c6b4 100644 --- a/src/frontend/specification/documentation.jl +++ b/src/frontend/specification/documentation.jl @@ -1,10 +1,3 @@ -""" - params(semobj) -> Vector{Symbol} - -Return the vector of SEM model parameter identifiers. -""" -function params end - params(spec::SemSpecification) = spec.params """ From 92b5741840caa9e5ff307529eff7944c95941b83 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:01:45 -0700 Subject: [PATCH 47/71] RAM ctor: use random parameters instead of NaNs to initialize RAM matrices simplify check_acyclic() --- src/imply/RAM/generic.jl | 41 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 850934a9c..69cbc517d 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -126,13 +126,11 @@ function RAM(; n_var = nvars(ram_matrices) #preallocate arrays - nan_params = fill(NaN, n_par) - A_pre = materialize(ram_matrices.A, nan_params) - S_pre = materialize(ram_matrices.S, nan_params) + rand_params = randn(Float64, n_par) + A_pre = check_acyclic(materialize(ram_matrices.A, rand_params)) + S_pre = materialize(ram_matrices.S, rand_params) F = copy(ram_matrices.F) - A_pre = check_acyclic(A_pre, ram_matrices.A) - # pre-allocate some matrices Σ = zeros(n_obs, n_obs) F⨉I_A⁻¹ = zeros(n_obs, n_var) @@ -155,7 +153,7 @@ function RAM(; "You set `meanstructure = true`, but your model specification contains no mean parameters.", ), ) - M_pre = materialize(ram_matrices.M, nan_params) + M_pre = materialize(ram_matrices.M, rand_params) ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else @@ -229,22 +227,21 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) - # fill copy of A with random parameters - A_rand = materialize(A, rand(nparams(A))) - - # check if the model is acyclic - acyclic = isone(det(I - A_rand)) - +# checks if the A matrix is acyclic +# wraps A in LowerTriangular/UpperTriangular if it is triangular +function check_acyclic(A::AbstractMatrix) # check if A is lower or upper triangular - if istril(A_rand) - A_pre = LowerTriangular(A_pre) - elseif istriu(A_rand) - A_pre = UpperTriangular(A_pre) - elseif acyclic - @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog = - 1 + if istril(A) + return LowerTriangular(A) + elseif istriu(A) + return UpperTriangular(A) + else + # check if non-triangular matrix is acyclic + acyclic = isone(det(I - A)) + if acyclic + @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 + end + return A end - - return A_pre end From ae2291a3fa8342836f0e16e5904b93dc8219aa29 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 13 Jun 2024 00:13:08 -0700 Subject: [PATCH 48/71] move check_acyclic() to abstract.jl add verbose parameter --- src/imply/RAM/generic.jl | 23 ----------------------- src/imply/abstract.jl | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 69cbc517d..56960e4ff 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -222,26 +222,3 @@ function update_observed(imply::RAM, observed::SemObserved; kwargs...) return RAM(; observed = observed, kwargs...) end end - -############################################################################################ -### additional functions -############################################################################################ - -# checks if the A matrix is acyclic -# wraps A in LowerTriangular/UpperTriangular if it is triangular -function check_acyclic(A::AbstractMatrix) - # check if A is lower or upper triangular - if istril(A) - return LowerTriangular(A) - elseif istriu(A) - return UpperTriangular(A) - else - # check if non-triangular matrix is acyclic - acyclic = isone(det(I - A)) - if acyclic - @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = - 1 - end - return A - end -end diff --git a/src/imply/abstract.jl b/src/imply/abstract.jl index 6a3f84191..37834415d 100644 --- a/src/imply/abstract.jl +++ b/src/imply/abstract.jl @@ -10,3 +10,24 @@ nlatent_vars(imply::SemImply) = nlatent_vars(imply.ram_matrices) params(imply::SemImply) = params(imply.ram_matrices) nparams(imply::SemImply) = nparams(imply.ram_matrices) + +# checks if the A matrix is acyclic +# wraps A in LowerTriangular/UpperTriangular if it is triangular +function check_acyclic(A::AbstractMatrix; verbose::Bool = false) + # check if A is lower or upper triangular + if istril(A) + verbose && @info "A matrix is lower triangular" + return LowerTriangular(A) + elseif istriu(A) + verbose && @info "A matrix is upper triangular" + return UpperTriangular(A) + else + # check if non-triangular matrix is acyclic + acyclic = isone(det(I - A)) + if acyclic + verbose && @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 + end + return A + end +end From 317525774e812ef1549c44fa7d0203b89013af2a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Dec 2024 12:44:19 -0800 Subject: [PATCH 49/71] AbstractSem: improve imply/observed API redirect --- src/frontend/specification/Sem.jl | 40 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 28984dbe9..d9b4a6e4e 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -20,45 +20,43 @@ function Sem(; return sem end -nvars(sem::AbstractSemSingle) = nvars(sem.imply) -nobserved_vars(sem::AbstractSemSingle) = nobserved_vars(sem.imply) -nlatent_vars(sem::AbstractSemSingle) = nlatent_vars(sem.imply) +""" + imply(model::AbstractSemSingle) -> SemImply -vars(sem::AbstractSemSingle) = vars(sem.imply) -observed_vars(sem::AbstractSemSingle) = observed_vars(sem.imply) -latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply) +Returns the [*implied*](@ref SemImply) part of a model. +""" +imply(model::AbstractSemSingle) = model.imply -nsamples(sem::AbstractSemSingle) = nsamples(sem.observed) +nvars(model::AbstractSemSingle) = nvars(imply(model)) +nobserved_vars(model::AbstractSemSingle) = nobserved_vars(imply(model)) +nlatent_vars(model::AbstractSemSingle) = nlatent_vars(imply(model)) -params(model::AbstractSem) = params(model.imply) +vars(model::AbstractSemSingle) = vars(imply(model)) +observed_vars(model::AbstractSemSingle) = observed_vars(imply(model)) +latent_vars(model::AbstractSemSingle) = latent_vars(imply(model)) -# sum of samples in all sub-models -nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) +params(model::AbstractSemSingle) = params(imply(model)) +nparams(model::AbstractSemSingle) = nparams(imply(model)) -############################################################################################ -# additional methods -############################################################################################ """ observed(model::AbstractSemSingle) -> SemObserved -Returns the observed part of a model. +Returns the [*observed*](@ref SemObserved) part of a model. """ observed(model::AbstractSemSingle) = model.observed -""" - imply(model::AbstractSemSingle) -> SemImply - -Returns the imply part of a model. -""" -imply(model::AbstractSemSingle) = model.imply +nsamples(model::AbstractSemSingle) = nsamples(observed(model)) """ loss(model::AbstractSemSingle) -> SemLoss -Returns the loss part of a model. +Returns the [*loss*](@ref SemLoss) function of a model. """ loss(model::AbstractSemSingle) = model.loss +# sum of samples in all sub-models +nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) + function SemFiniteDiff(; specification = ParameterTable, observed::O = SemObservedData, From c8b1645f118576c7835b6a486888285682f285a6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Thu, 9 Jan 2025 14:24:53 -0800 Subject: [PATCH 50/71] imply -> implied, SemImply -> SemImplied --- docs/src/developer/imply.md | 38 ++++++------ docs/src/developer/loss.md | 10 ++-- docs/src/developer/observed.md | 2 +- docs/src/developer/sem.md | 14 ++--- docs/src/internals/files.md | 2 +- docs/src/internals/types.md | 2 +- docs/src/performance/symbolic.md | 2 +- docs/src/tutorials/concept.md | 26 ++++---- .../tutorials/construction/build_by_parts.md | 8 +-- .../construction/outer_constructor.md | 12 ++-- docs/src/tutorials/fitting/fitting.md | 2 +- docs/src/tutorials/meanstructure.md | 4 +- src/StructuralEquationModels.jl | 8 +-- src/additional_functions/helper.jl | 8 +-- src/additional_functions/simulation.jl | 30 +++++----- .../start_val/start_fabin3.jl | 10 ++-- .../start_val/start_simple.jl | 6 +- src/frontend/fit/fitmeasures/chi2.jl | 2 +- src/frontend/fit/fitmeasures/df.jl | 2 +- src/frontend/fit/fitmeasures/minus2ll.jl | 4 +- src/frontend/pretty_printing.jl | 4 +- src/frontend/specification/Sem.jl | 60 +++++++++---------- src/imply/RAM/generic.jl | 56 +++++++---------- src/imply/RAM/symbolic.jl | 26 ++++---- src/imply/abstract.jl | 18 +++--- src/imply/empty.jl | 18 +++--- src/loss/ML/FIML.jl | 28 ++++----- src/loss/ML/ML.jl | 8 +-- src/loss/WLS/WLS.jl | 2 +- src/loss/regularization/ridge.jl | 14 ++--- src/objective_gradient_hessian.jl | 44 +++++++------- src/optimizer/abstract.jl | 2 +- src/types.jl | 42 ++++++------- test/examples/multigroup/build_models.jl | 26 ++++---- test/examples/political_democracy/by_parts.jl | 38 ++++++------ .../political_democracy/constructor.jl | 16 ++--- .../recover_parameters_twofact.jl | 8 +-- test/unit_tests/model.jl | 10 ++-- test/unit_tests/sorting.jl | 2 +- 39 files changed, 299 insertions(+), 315 deletions(-) diff --git a/docs/src/developer/imply.md b/docs/src/developer/imply.md index cb30e40fe..403ecfa84 100644 --- a/docs/src/developer/imply.md +++ b/docs/src/developer/imply.md @@ -1,11 +1,11 @@ -# Custom imply types +# Custom implied types We recommend to first read the part [Custom loss functions](@ref), as the overall implementation is the same and we will describe it here more briefly. -Imply types are of subtype `SemImply`. To implement your own imply type, you should define a struct +Implied types are of subtype `SemImplied`. To implement your own implied type, you should define a struct ```julia -struct MyImply <: SemImply +struct MyImplied <: SemImplied ... end ``` @@ -15,37 +15,37 @@ and at least a method to compute the objective ```julia import StructuralEquationModels: objective! -function objective!(imply::MyImply, par, model::AbstractSemSingle) +function objective!(implied::MyImplied, par, model::AbstractSemSingle) ... return nothing end ``` -This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` imply type computes the model-implied covariance matrix and makes it available via `Σ(imply)`. -To make stored computations available to loss functions, simply write a function - for example, for the `RAM` imply type we defined +This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `Σ(implied)`. +To make stored computations available to loss functions, simply write a function - for example, for the `RAM` implied type we defined ```julia -Σ(imply::RAM) = imply.Σ +Σ(implied::RAM) = implied.Σ ``` Additionally, you can specify methods for `gradient` and `hessian` as well as the combinations described in [Custom loss functions](@ref). -The last thing nedded to make it work is a method for `nparams` that takes your imply type and returns the number of parameters of the model: +The last thing nedded to make it work is a method for `nparams` that takes your implied type and returns the number of parameters of the model: ```julia -nparams(imply::MyImply) = ... +nparams(implied::MyImplied) = ... ``` Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. -We implement an `ImplyEmpty` type in our package that does nothing but serving as an imply field in case you are using a loss function that does not need any imply type at all. You may use it as a template for defining your own imply type, as it also shows how to handle the specification objects: +We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: ```julia ############################################################################ ### Types ############################################################################ -struct ImplyEmpty{V, V2} <: SemImply +struct ImpliedEmpty{V, V2} <: SemImplied identifier::V2 n_par::V end @@ -54,7 +54,7 @@ end ### Constructors ############################################################################ -function ImplyEmpty(; +function ImpliedEmpty(; specification, kwargs...) @@ -63,25 +63,25 @@ function ImplyEmpty(; n_par = length(ram_matrices.parameters) - return ImplyEmpty(identifier, n_par) + return ImpliedEmpty(identifier, n_par) end ############################################################################ ### methods ############################################################################ -objective!(imply::ImplyEmpty, par, model) = nothing -gradient!(imply::ImplyEmpty, par, model) = nothing -hessian!(imply::ImplyEmpty, par, model) = nothing +objective!(implied::ImpliedEmpty, par, model) = nothing +gradient!(implied::ImpliedEmpty, par, model) = nothing +hessian!(implied::ImpliedEmpty, par, model) = nothing ############################################################################ ### Recommended methods ############################################################################ -identifier(imply::ImplyEmpty) = imply.identifier -n_par(imply::ImplyEmpty) = imply.n_par +identifier(implied::ImpliedEmpty) = implied.identifier +n_par(implied::ImpliedEmpty) = implied.n_par -update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply +update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied ``` As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. Additionally, you should store the `identifier` from the specification object and write a method for `identifier`, as this will make it possible to access parameter indices by label. \ No newline at end of file diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index e1137dbf1..4f42a4700 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -171,7 +171,7 @@ function MyLoss(;arg1 = ..., arg2, kwargs...) end ``` -All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (imply and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector: +All keyword arguments that a user passes to the Sem constructor are passed to your loss function. In addition, all previously constructed parts of the model (implied and observed part) are passed as keyword arguments as well as the number of parameters `n_par = ...`, so your constructor may depend on those. For example, the constructor for `SemML` in our package depends on the additional argument `meanstructure` as well as the observed part of the model to pre-allocate arrays of the same size as the observed covariance matrix and the observed mean vector: ```julia function SemML(;observed, meanstructure = false, approx_H = false, kwargs...) @@ -221,9 +221,9 @@ To keep it simple, we only cover models without a meanstructure. The maximum lik F_{ML} = \log \det \Sigma_i + \mathrm{tr}\left(\Sigma_{i}^{-1} \Sigma_o \right) ``` -where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `imply` par of our model, and the observed covariance matrix from the `observed` path of our model. +where ``\Sigma_i`` is the model implied covariance matrix and ``\Sigma_o`` is the observed covariance matrix. We can query the model implied covariance matrix from the `implied` par of our model, and the observed covariance matrix from the `observed` path of our model. -To get information on what we can access from a certain `imply` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL: +To get information on what we can access from a certain `implied` or `observed` type, we can check it`s documentation an the pages [API - model parts](@ref) or via the help mode of the REPL: ```julia julia>? @@ -233,7 +233,7 @@ help?> RAM help?> SemObservedCommon ``` -We see that the model implied covariance matrix can be assessed as `Σ(imply)` and the observed covariance matrix as `obs_cov(observed)`. +We see that the model implied covariance matrix can be assessed as `Σ(implied)` and the observed covariance matrix as `obs_cov(observed)`. With this information, we write can implement maximum likelihood optimization as @@ -245,7 +245,7 @@ import StructuralEquationModels: Σ, obs_cov, objective! function objective!(semml::MaximumLikelihood, parameters, model::AbstractSem) # access the model implied and observed covariance matrices - Σᵢ = Σ(imply(model)) + Σᵢ = Σ(implied(model)) Σₒ = obs_cov(observed(model)) # compute the objective if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite? diff --git a/docs/src/developer/observed.md b/docs/src/developer/observed.md index 93eca6ed9..240c1c34f 100644 --- a/docs/src/developer/observed.md +++ b/docs/src/developer/observed.md @@ -28,7 +28,7 @@ nsamples(observed::MyObserved) = ... nobserved_vars(observed::MyObserved) = ... ``` -As always, you can add additional methods for properties that imply types and loss function want to access, for example (from the `SemObservedCommon` implementation): +As always, you can add additional methods for properties that implied types and loss function want to access, for example (from the `SemObservedCommon` implementation): ```julia obs_cov(observed::SemObservedCommon) = observed.obs_cov diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index 528da88b8..0063a85cf 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -1,15 +1,15 @@ # Custom model types -The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, imply, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as +The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as ```julia struct SemFiniteDiff{ - O <: SemObserved, - I <: SemImply, - L <: SemLoss, + O <: SemObserved, + I <: SemImplied, + L <: SemLoss, D <: SemOptimizer} <: AbstractSemSingle{O, I, L, D} observed::O - imply::I + implied::I loss::L optimizer::D end @@ -19,13 +19,13 @@ Additionally, we need to define a method to compute at least the objective value ```julia function objective!(model::AbstractSemSingle, parameters) - objective!(imply(model), parameters, model) + objective!(implied(model), parameters, model) return objective!(loss(model), parameters, model) end function gradient!(gradient, model::AbstractSemSingle, parameters) fill!(gradient, zero(eltype(gradient))) - gradient!(imply(model), parameters, model) + gradient!(implied(model), parameters, model) gradient!(gradient, loss(model), parameters, model) end ``` diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 06c73444d..9cf455fdc 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -10,7 +10,7 @@ All source code is in the `"src"` folder: - `"StructuralEquationModels.jl"` defines the module and the exported objects - `"types.jl"` defines all abstract types and the basic type hierarchy - `"objective_gradient_hessian.jl"` contains methods for computing objective, gradient and hessian values for different model types as well as generic fallback methods -- The four folders `"observed"`, `"imply"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). +- The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). - `"optimizer"` contains connections to different optimization backends (aka methods for `sem_fit`) - `"optim.jl"`: connection to the `Optim.jl` package - `"NLopt.jl"`: connection to the `NLopt.jl` package diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index 488127b29..980d0f42f 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -8,6 +8,6 @@ The type hierarchy is implemented in `"src/types.jl"`. - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation - `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels -Every `AbstractSemSingle` has to have `SemObserved`, `SemImply`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). +Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). `SemLoss` is a container for multiple `SemLossFunctions`. \ No newline at end of file diff --git a/docs/src/performance/symbolic.md b/docs/src/performance/symbolic.md index 597d2c484..05729526e 100644 --- a/docs/src/performance/symbolic.md +++ b/docs/src/performance/symbolic.md @@ -13,6 +13,6 @@ If the model is acyclic, we can compute ``` for some ``n < \infty``. -Typically, the ``S`` and ``A`` matrices are sparse. In our package, we offer symbolic precomputation of ``\Sigma``, ``\nabla\Sigma`` and even ``\nabla^2\Sigma`` for acyclic models to optimally exploit this sparsity. To use this feature, simply use the `RAMSymbolic` imply type for your model. +Typically, the ``S`` and ``A`` matrices are sparse. In our package, we offer symbolic precomputation of ``\Sigma``, ``\nabla\Sigma`` and even ``\nabla^2\Sigma`` for acyclic models to optimally exploit this sparsity. To use this feature, simply use the `RAMSymbolic` implied type for your model. This can decrase model fitting time, but will also increase model building time (as we have to carry out the symbolic computations and compile specialised functions). As a result, this is probably not beneficial to use if you only fit a single model, but can lead to great improvements if you fit the same modle to multiple datasets (e.g. to compute bootstrap standard errors). \ No newline at end of file diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index c63c15941..b8d094abc 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -4,9 +4,9 @@ In our package, every Structural Equation Model (`Sem`) consists of four parts: ![SEM concept](../assets/concept.svg) -Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the 'observed' slot of the model, and stick them together with other pieces that can serve as the 'imply' part. +Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part. -The 'observed' part is for observed data, the imply part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). +The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer. @@ -20,24 +20,24 @@ So everything that can be used as the 'observed' part has to be of type `SemObse Here is an overview on the available building blocks: -|[`SemObserved`](@ref) | [`SemImply`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | +|[`SemObserved`](@ref) | [`SemImplied`](@ref) | [`SemLossFunction`](@ref) | [`SemOptimizer`](@ref) | |---------------------------------|-----------------------|---------------------------|-------------------------------| | [`SemObservedData`](@ref) | [`RAM`](@ref) | [`SemML`](@ref) | [`SemOptimizerOptim`](@ref) | | [`SemObservedCovariance`](@ref) | [`RAMSymbolic`](@ref) | [`SemWLS`](@ref) | [`SemOptimizerNLopt`](@ref) | -| [`SemObservedMissing`](@ref) | [`ImplyEmpty`](@ref) | [`SemFIML`](@ref) | | -| | | [`SemRidge`](@ref) | | -| | | [`SemConstant`](@ref) | | +| [`SemObservedMissing`](@ref) | [`ImpliedEmpty`](@ref)| [`SemFIML`](@ref) | | +| | | [`SemRidge`](@ref) | | +| | | [`SemConstant`](@ref) | | The rest of this page explains the building blocks for each part. First, we explain every part and give an overview on the different options that are available. After that, the [API - model parts](@ref) section serves as a reference for detailed explanations about the different options. (How to stick them together to a final model is explained in the section on [Model Construction](@ref).) ## The observed part aka [`SemObserved`](@ref) -The 'observed' part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values. +The *observed* part contains all necessary information about the observed data. Currently, we have three options: [`SemObservedData`](@ref) for fully observed datasets, [`SemObservedCovariance`](@ref) for observed covariances (and means) and [`SemObservedMissing`](@ref) for data that contains missing values. -## The imply part aka [`SemImply`](@ref) -The imply part is what your model implies about the data, for example, the model-implied covariance matrix. -There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImplyEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an imply part. +## The implied part aka [`SemImplied`](@ref) +The *implied* part is what your model implies about the data, for example, the model-implied covariance matrix. +There are two options at the moment: [`RAM`](@ref), which uses the reticular action model to compute the model implied covariance matrix, and [`RAMSymbolic`](@ref) which does the same but symbolically pre-computes part of the model, which increases subsequent performance in model fitting (see [Symbolic precomputation](@ref)). There is also a third option, [`ImpliedEmpty`](@ref) that can serve as a 'placeholder' for models that do not need an implied part. ## The loss part aka `SemLoss` The loss part specifies the objective that is optimized to find the parameter estimates. @@ -73,13 +73,13 @@ SemObservedCovariance SemObservedMissing ``` -## imply +## implied ```@docs -SemImply +SemImplied RAM RAMSymbolic -ImplyEmpty +ImpliedEmpty ``` ## loss functions diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 779949d98..071750a8c 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -1,6 +1,6 @@ # Build by parts -You can always build a model by parts - that is, you construct the observed, imply, loss and optimizer part seperately. +You can always build a model by parts - that is, you construct the observed, implied, loss and optimizer part seperately. As an example on how this works, we will build [A first model](@ref) in parts. @@ -50,8 +50,8 @@ Now, we construct the different parts: # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = partable, data = data) -# imply ------------------------------------------------------------------------------------ -imply_ram = RAM(specification = partable) +# implied ------------------------------------------------------------------------------------ +implied_ram = RAM(specification = partable) # loss ------------------------------------------------------------------------------------- ml = SemML(observed = observed) @@ -63,5 +63,5 @@ optimizer = SemOptimizerOptim() # model ------------------------------------------------------------------------------------ -model_ml = Sem(observed, imply_ram, loss_ml, optimizer) +model_ml = Sem(observed, implied_ram, loss_ml, optimizer) ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index f072b80bc..0979f684a 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -15,13 +15,13 @@ Structural Equation Model SemML - Fields observed: SemObservedCommon - imply: RAM + implied: RAM optimizer: SemOptimizerOptim ``` -The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, imply and optimizer parts are). +The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). -As you can see, by default, we use maximum likelihood estimation, the RAM imply type and the `Optim.jl` optimization backend. +As you can see, by default, we use maximum likelihood estimation, the RAM implied type and the `Optim.jl` optimization backend. To choose something different, you can provide it as a keyword argument: ```julia @@ -29,7 +29,7 @@ model = Sem( specification = partable, data = data, observed = ..., - imply = ..., + implied = ..., loss = ..., optimizer = ... ) @@ -41,7 +41,7 @@ For example, to construct a model for weighted least squares estimation that use model = Sem( specification = partable, data = data, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, optimizer = SemOptimizerNLopt ) @@ -73,7 +73,7 @@ W = ... model = Sem( specification = partable, data = data, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, wls_weight_matrix = W ) diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index f78a6c0db..b534ad754 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -16,7 +16,7 @@ Structural Equation Model SemML - Fields observed: SemObservedData - imply: RAM + implied: RAM optimizer: SemOptimizerOptim ------------- Optimization result ------------- diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index c6ad692b6..692f6cebc 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -106,11 +106,11 @@ For our example, ```@example meanstructure observed = SemObservedData(specification = partable, data = data, meanstructure = true) -imply_ram = RAM(specification = partable, meanstructure = true) +implied_ram = RAM(specification = partable, meanstructure = true) ml = SemML(observed = observed, meanstructure = true) -model = Sem(observed, imply_ram, SemLoss(ml), SemOptimizerOptim()) +model = Sem(observed, implied_ram, SemLoss(ml), SemOptimizerOptim()) sem_fit(model) ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..a9a5af0d7 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -49,7 +49,7 @@ include("observed/EM.jl") # constructor include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") -# imply +# implied include("imply/abstract.jl") include("imply/RAM/symbolic.jl") include("imply/RAM/generic.jl") @@ -95,11 +95,11 @@ export AbstractSem, HessianEval, ExactHessian, ApproxHessian, - SemImply, + SemImplied, RAMSymbolic, RAM, - ImplyEmpty, - imply, + ImpliedEmpty, + implied, start_val, start_fabin3, start_simple, diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 71b2559a8..5559034e0 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -21,14 +21,14 @@ function make_onelement_array(A) end =# -function semvec(observed, imply, loss, optimizer) +function semvec(observed, implied, loss, optimizer) observed = make_onelement_array(observed) - imply = make_onelement_array(imply) + implied = make_onelement_array(implied) loss = make_onelement_array(loss) optimizer = make_onelement_array(optimizer) - #sem_vec = Array{AbstractSem}(undef, maximum(length.([observed, imply, loss, optimizer]))) - sem_vec = Sem.(observed, imply, loss, optimizer) + #sem_vec = Array{AbstractSem}(undef, maximum(length.([observed, implied, loss, optimizer]))) + sem_vec = Sem.(observed, implied, loss, optimizer) return sem_vec end diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 0b2626b15..8c1a093a6 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -18,7 +18,7 @@ function swap_observed end """ update_observed(to_update, observed::SemObserved; kwargs...) -Update a `SemImply`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. +Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. # Examples See the online documentation on [Swap observed data](@ref). @@ -45,7 +45,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = swap_observed( model, observed(model), - imply(model), + implied(model), loss(model), new_observed; kwargs..., @@ -54,7 +54,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = function swap_observed( model::AbstractSemSingle, old_observed, - imply, + implied, loss, new_observed::SemObserved; kwargs..., @@ -64,23 +64,23 @@ function swap_observed( # get field types kwargs[:observed_type] = typeof(new_observed) kwargs[:old_observed_type] = typeof(old_observed) - kwargs[:imply_type] = typeof(imply) + kwargs[:implied_type] = typeof(implied) kwargs[:loss_types] = [typeof(lossfun) for lossfun in loss.functions] - # update imply - imply = update_observed(imply, new_observed; kwargs...) - kwargs[:imply] = imply - kwargs[:nparams] = nparams(imply) + # update implied + implied = update_observed(implied, new_observed; kwargs...) + kwargs[:implied] = implied + kwargs[:nparams] = nparams(implied) # update loss loss = update_observed(loss, new_observed; kwargs...) kwargs[:loss] = loss - #new_imply = update_observed(model.imply, new_observed; kwargs...) + #new_implied = update_observed(model.implied, new_observed; kwargs...) return Sem( new_observed, - update_observed(model.imply, new_observed; kwargs...), + update_observed(model.implied, new_observed; kwargs...), update_observed(model.loss, new_observed; kwargs...), ) end @@ -117,7 +117,7 @@ function Distributions.rand( params, n::Integer, ) where {O, I <: Union{RAM, RAMSymbolic}, L} - update!(EvaluationTargets{true, false, false}(), model.imply, model, params) + update!(EvaluationTargets{true, false, false}(), model.implied, model, params) return rand(model, n) end @@ -125,10 +125,10 @@ function Distributions.rand( model::AbstractSemSingle{O, I, L}, n::Integer, ) where {O, I <: Union{RAM, RAMSymbolic}, L} - if MeanStruct(model.imply) === NoMeanStruct - data = permutedims(rand(MvNormal(Symmetric(model.imply.Σ)), n)) - elseif MeanStruct(model.imply) === HasMeanStruct - data = permutedims(rand(MvNormal(model.imply.μ, Symmetric(model.imply.Σ)), n)) + if MeanStruct(model.implied) === NoMeanStruct + data = permutedims(rand(MvNormal(Symmetric(model.implied.Σ)), n)) + elseif MeanStruct(model.implied) === HasMeanStruct + data = permutedims(rand(MvNormal(model.implied.μ, Symmetric(model.implied.Σ)), n)) end return data end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index dd8d61fd9..bd55f21d7 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -8,20 +8,20 @@ function start_fabin3 end # splice model and loss functions function start_fabin3(model::AbstractSemSingle; kwargs...) - return start_fabin3(model.observed, model.imply, model.loss.functions..., kwargs...) + return start_fabin3(model.observed, model.implied, model.loss.functions..., kwargs...) end -function start_fabin3(observed, imply, args...; kwargs...) - return start_fabin3(imply.ram_matrices, obs_cov(observed), obs_mean(observed)) +function start_fabin3(observed, implied, args...; kwargs...) + return start_fabin3(implied.ram_matrices, obs_cov(observed), obs_mean(observed)) end # SemObservedMissing -function start_fabin3(observed::SemObservedMissing, imply, args...; kwargs...) +function start_fabin3(observed::SemObservedMissing, implied, args...; kwargs...) if !observed.em_model.fitted em_mvn(observed; kwargs...) end - return start_fabin3(imply.ram_matrices, observed.em_model.Σ, observed.em_model.μ) + return start_fabin3(implied.ram_matrices, observed.em_model.Σ, observed.em_model.μ) end function start_fabin3( diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 1f16b094c..ad5148e3f 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -17,11 +17,11 @@ function start_simple end # Single Models ---------------------------------------------------------------------------- function start_simple(model::AbstractSemSingle; kwargs...) - return start_simple(model.observed, model.imply, model.loss.functions...; kwargs...) + return start_simple(model.observed, model.implied, model.loss.functions...; kwargs...) end -function start_simple(observed, imply, args...; kwargs...) - return start_simple(imply.ram_matrices; kwargs...) +function start_simple(observed, implied, args...; kwargs...) + return start_simple(implied.ram_matrices; kwargs...) end # Ensemble Models -------------------------------------------------------------------------- diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 12bc1d880..333783f95 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -13,7 +13,7 @@ function χ² end χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = χ²( sem_fit, sem_fit.model.observed, - sem_fit.model.imply, + sem_fit.model.implied, sem_fit.model.loss.functions..., ) diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/df.jl index e8e72d594..4d9025601 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/df.jl @@ -13,7 +13,7 @@ df(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) nvars = nobserved_vars(model) ndp = 0.5(nvars^2 + nvars) - if !isnothing(model.imply.μ) + if !isnothing(model.implied.μ) ndp += nvars end return ndp diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 1cddee71d..2cb87d79c 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -15,7 +15,7 @@ minus2ll( ) = minus2ll( sem_fit, sem_fit.model.observed, - sem_fit.model.imply, + sem_fit.model.implied, sem_fit.model.loss.functions..., ) @@ -67,7 +67,7 @@ end ############################################################################################ minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.imply, model.loss.functions...) + minus2ll(minimum, model.observed, model.implied, model.loss.functions...) function minus2ll( sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}, diff --git a/src/frontend/pretty_printing.jl b/src/frontend/pretty_printing.jl index 5b732c980..c1cd72c2f 100644 --- a/src/frontend/pretty_printing.jl +++ b/src/frontend/pretty_printing.jl @@ -25,7 +25,7 @@ function print_type(io::IO, struct_instance) end ############################################################## -# Loss Functions, Imply, +# Loss Functions, Implied, ############################################################## function Base.show(io::IO, struct_inst::SemLossFunction) @@ -33,7 +33,7 @@ function Base.show(io::IO, struct_inst::SemLossFunction) print_field_types(io, struct_inst) end -function Base.show(io::IO, struct_inst::SemImply) +function Base.show(io::IO, struct_inst::SemImplied) print_type_name(io, struct_inst) print_field_types(io, struct_inst) end diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index d9b4a6e4e..33440e257 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -5,38 +5,38 @@ function Sem(; specification = ParameterTable, observed::O = SemObservedData, - imply::I = RAM, + implied::I = RAM, loss::L = SemML, kwargs..., ) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) + set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) - observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) + observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) - sem = Sem(observed, imply, loss) + sem = Sem(observed, implied, loss) return sem end """ - imply(model::AbstractSemSingle) -> SemImply + implied(model::AbstractSemSingle) -> SemImplied -Returns the [*implied*](@ref SemImply) part of a model. +Returns the [*implied*](@ref SemImplied) part of a model. """ -imply(model::AbstractSemSingle) = model.imply +implied(model::AbstractSemSingle) = model.implied -nvars(model::AbstractSemSingle) = nvars(imply(model)) -nobserved_vars(model::AbstractSemSingle) = nobserved_vars(imply(model)) -nlatent_vars(model::AbstractSemSingle) = nlatent_vars(imply(model)) +nvars(model::AbstractSemSingle) = nvars(implied(model)) +nobserved_vars(model::AbstractSemSingle) = nobserved_vars(implied(model)) +nlatent_vars(model::AbstractSemSingle) = nlatent_vars(implied(model)) -vars(model::AbstractSemSingle) = vars(imply(model)) -observed_vars(model::AbstractSemSingle) = observed_vars(imply(model)) -latent_vars(model::AbstractSemSingle) = latent_vars(imply(model)) +vars(model::AbstractSemSingle) = vars(implied(model)) +observed_vars(model::AbstractSemSingle) = observed_vars(implied(model)) +latent_vars(model::AbstractSemSingle) = latent_vars(implied(model)) -params(model::AbstractSemSingle) = params(imply(model)) -nparams(model::AbstractSemSingle) = nparams(imply(model)) +params(model::AbstractSemSingle) = params(implied(model)) +nparams(model::AbstractSemSingle) = nparams(implied(model)) """ observed(model::AbstractSemSingle) -> SemObserved @@ -60,17 +60,17 @@ nsamples(ensemble::SemEnsemble) = sum(nsamples, ensemble.sems) function SemFiniteDiff(; specification = ParameterTable, observed::O = SemObservedData, - imply::I = RAM, + implied::I = RAM, loss::L = SemML, kwargs..., ) where {O, I, L} kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwdict, observed, imply, loss, O, I) + set_field_type_kwargs!(kwdict, observed, implied, loss, O, I) - observed, imply, loss = get_fields!(kwdict, specification, observed, imply, loss) + observed, implied, loss = get_fields!(kwdict, specification, observed, implied, loss) - sem = SemFiniteDiff(observed, imply, loss) + sem = SemFiniteDiff(observed, implied, loss) return sem end @@ -79,9 +79,9 @@ end # functions ############################################################################################ -function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) +function set_field_type_kwargs!(kwargs, observed, implied, loss, O, I) kwargs[:observed_type] = O <: Type ? observed : typeof(observed) - kwargs[:imply_type] = I <: Type ? imply : typeof(imply) + kwargs[:implied_type] = I <: Type ? implied : typeof(implied) if loss isa SemLoss kwargs[:loss_types] = [ lossfun isa SemLossFunction ? typeof(lossfun) : lossfun for @@ -96,7 +96,7 @@ function set_field_type_kwargs!(kwargs, observed, imply, loss, O, I) end # construct Sem fields -function get_fields!(kwargs, specification, observed, imply, loss) +function get_fields!(kwargs, specification, observed, implied, loss) if !isa(specification, SemSpecification) specification = specification(; kwargs...) end @@ -107,19 +107,19 @@ function get_fields!(kwargs, specification, observed, imply, loss) end kwargs[:observed] = observed - # imply - if !isa(imply, SemImply) - imply = imply(; specification, kwargs...) + # implied + if !isa(implied, SemImplied) + implied = implied(; specification, kwargs...) end - kwargs[:imply] = imply - kwargs[:nparams] = nparams(imply) + kwargs[:implied] = implied + kwargs[:nparams] = nparams(implied) # loss loss = get_SemLoss(loss; specification, kwargs...) kwargs[:loss] = loss - return observed, imply, loss + return observed, implied, loss end # construct loss field @@ -164,7 +164,7 @@ function Base.show(io::IO, sem::Sem{O, I, L}) where {O, I, L} print(io, lossfuntypes...) print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") - print(io, " imply: $(nameof(I)) \n") + print(io, " implied: $(nameof(I)) \n") end function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} @@ -175,7 +175,7 @@ function Base.show(io::IO, sem::SemFiniteDiff{O, I, L}) where {O, I, L} print(io, lossfuntypes...) print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") - print(io, " imply: $(nameof(I)) \n") + print(io, " implied: $(nameof(I)) \n") end function Base.show(io::IO, loss::SemLoss) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 56960e4ff..30bd29bf4 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -20,7 +20,7 @@ Model implied covariance and means via RAM notation. # Extended help ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. ## RAM notation @@ -65,23 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{ - MS, - A1, - A2, - A3, - A4, - A5, - A6, - V2, - M1, - M2, - M3, - M4, - S1, - S2, - S3, -} <: SemImply +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3} <: SemImplied meanstruct::MS hessianeval::ExactHessian @@ -185,29 +169,29 @@ end ### methods ############################################################################################ -function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) - materialize!(imply.A, imply.ram_matrices.A, params) - materialize!(imply.S, imply.ram_matrices.S, params) - if !isnothing(imply.M) - materialize!(imply.M, imply.ram_matrices.M, params) +function update!(targets::EvaluationTargets, implied::RAM, model::AbstractSemSingle, params) + materialize!(implied.A, implied.ram_matrices.A, params) + materialize!(implied.S, implied.ram_matrices.S, params) + if !isnothing(implied.M) + materialize!(implied.M, implied.ram_matrices.M, params) end - @. imply.I_A = -imply.A - @view(imply.I_A[diagind(imply.I_A)]) .+= 1 + parent(implied.I_A) .= .-implied.A + @view(implied.I_A[diagind(implied.I_A)]) .+= 1 if is_gradient_required(targets) || is_hessian_required(targets) - imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) - mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) + implied.I_A⁻¹ = LinearAlgebra.inv!(factorize(implied.I_A)) + mul!(implied.F⨉I_A⁻¹, implied.F, implied.I_A⁻¹) else - copyto!(imply.F⨉I_A⁻¹, imply.F) - rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) + copyto!(implied.F⨉I_A⁻¹, implied.F) + rdiv!(implied.F⨉I_A⁻¹, factorize(implied.I_A)) end - mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) - mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') + mul!(implied.F⨉I_A⁻¹S, implied.F⨉I_A⁻¹, implied.S) + mul!(parent(implied.Σ), implied.F⨉I_A⁻¹S, implied.F⨉I_A⁻¹') - if MeanStruct(imply) === HasMeanStruct - mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) + if MeanStruct(implied) === HasMeanStruct + mul!(implied.μ, implied.F⨉I_A⁻¹, implied.M) end end @@ -215,9 +199,9 @@ end ### Recommended methods ############################################################################################ -function update_observed(imply::RAM, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(imply.Σ, 1) - return imply +function update_observed(implied::RAM, observed::SemObserved; kwargs...) + if nobserved_vars(observed) == size(implied.Σ, 1) + return implied else return RAM(; observed = observed, kwargs...) end diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 32ffcc068..07acef019 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -2,7 +2,7 @@ ### Types ############################################################################################ @doc raw""" -Subtype of `SemImply` that implements the RAM notation with symbolic precomputation. +Subtype of `SemImplied` that implements the RAM notation with symbolic precomputation. # Constructor @@ -26,7 +26,7 @@ Subtype of `SemImply` that implements the RAM notation with symbolic precomputat # Extended help ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. ## Interfaces - `params(::RAMSymbolic) `-> vector of parameter ids @@ -63,7 +63,7 @@ and for models with a meanstructure, the model implied means are computed as ``` """ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: - SemImplySymbolic + SemImpliedSymbolic meanstruct::MS hessianeval::ExactHessian Σ_function::F1 @@ -201,19 +201,19 @@ end function update!( targets::EvaluationTargets, - imply::RAMSymbolic, + implied::RAMSymbolic, model::AbstractSemSingle, par, ) - imply.Σ_function(imply.Σ, par) - if MeanStruct(imply) === HasMeanStruct - imply.μ_function(imply.μ, par) + implied.Σ_function(implied.Σ, par) + if MeanStruct(implied) === HasMeanStruct + implied.μ_function(implied.μ, par) end if is_gradient_required(targets) || is_hessian_required(targets) - imply.∇Σ_function(imply.∇Σ, par) - if MeanStruct(imply) === HasMeanStruct - imply.∇μ_function(imply.∇μ, par) + implied.∇Σ_function(implied.∇Σ, par) + if MeanStruct(implied) === HasMeanStruct + implied.∇μ_function(implied.∇μ, par) end end end @@ -222,9 +222,9 @@ end ### Recommended methods ############################################################################################ -function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) - if nobserved_vars(observed) == size(imply.Σ, 1) - return imply +function update_observed(implied::RAMSymbolic, observed::SemObserved; kwargs...) + if nobserved_vars(observed) == size(implied.Σ, 1) + return implied else return RAMSymbolic(; observed = observed, kwargs...) end diff --git a/src/imply/abstract.jl b/src/imply/abstract.jl index 37834415d..05b0e2449 100644 --- a/src/imply/abstract.jl +++ b/src/imply/abstract.jl @@ -1,15 +1,15 @@ -# vars and params API methods for SemImply -vars(imply::SemImply) = vars(imply.ram_matrices) -observed_vars(imply::SemImply) = observed_vars(imply.ram_matrices) -latent_vars(imply::SemImply) = latent_vars(imply.ram_matrices) +# vars and params API methods for SemImplied +vars(implied::SemImplied) = vars(implied.ram_matrices) +observed_vars(implied::SemImplied) = observed_vars(implied.ram_matrices) +latent_vars(implied::SemImplied) = latent_vars(implied.ram_matrices) -nvars(imply::SemImply) = nvars(imply.ram_matrices) -nobserved_vars(imply::SemImply) = nobserved_vars(imply.ram_matrices) -nlatent_vars(imply::SemImply) = nlatent_vars(imply.ram_matrices) +nvars(implied::SemImplied) = nvars(implied.ram_matrices) +nobserved_vars(implied::SemImplied) = nobserved_vars(implied.ram_matrices) +nlatent_vars(implied::SemImplied) = nlatent_vars(implied.ram_matrices) -params(imply::SemImply) = params(imply.ram_matrices) -nparams(imply::SemImply) = nparams(imply.ram_matrices) +params(implied::SemImplied) = params(implied.ram_matrices) +nparams(implied::SemImplied) = nparams(implied.ram_matrices) # checks if the A matrix is acyclic # wraps A in LowerTriangular/UpperTriangular if it is triangular diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 66373bc1b..e87dc72d1 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -2,19 +2,19 @@ ### Types ############################################################################################ """ -Empty placeholder for models that don't need an imply part. +Empty placeholder for models that don't need an implied part. (For example, models that only regularize parameters.) # Constructor - ImplyEmpty(;specification, kwargs...) + ImpliedEmpty(;specification, kwargs...) # Arguments - `specification`: either a `RAMMatrices` or `ParameterTable` object # Examples A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one -model per group and an additional model with `ImplyEmpty` and `SemRidge` for the regularization part. +model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. # Extended help @@ -23,9 +23,9 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the - `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation -Subtype of `SemImply`. +Subtype of `SemImplied`. """ -struct ImplyEmpty{V2} <: SemImply +struct ImpliedEmpty{V2} <: SemImplied hessianeval::ExactHessian meanstruct::NoMeanStruct ram_matrices::V2 @@ -35,18 +35,18 @@ end ### Constructors ############################################################################################ -function ImplyEmpty(; specification, kwargs...) - return ImplyEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) +function ImpliedEmpty(; specification, kwargs...) + return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end ############################################################################################ ### methods ############################################################################################ -update!(targets::EvaluationTargets, imply::ImplyEmpty, par, model) = nothing +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply +update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index bf020d561..2c398090a 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -93,7 +93,7 @@ function evaluate!( gradient, hessian, semfiml::SemFIML, - implied::SemImply, + implied::SemImplied, model::AbstractSemSingle, params, ) @@ -148,20 +148,20 @@ function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, mo end end -function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) - mul!(G, imply.∇Σ', JΣ) # should be transposed - mul!(G, imply.∇μ', Jμ, -1, 1) +function ∇F_fiml_outer!(G, JΣ, Jμ, implied::SemImpliedSymbolic, model, semfiml) + mul!(G, implied.∇Σ', JΣ) # should be transposed + mul!(G, implied.∇μ', Jμ, -1, 1) end -function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) - Iₙ = sparse(1.0I, size(imply.A)...) - P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) - Q = kron(imply.S * imply.I_A⁻¹', Iₙ) +function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) + Iₙ = sparse(1.0I, size(implied.A)...) + P = kron(implied.F⨉I_A⁻¹, implied.F⨉I_A⁻¹) + Q = kron(implied.S * implied.I_A⁻¹', Iₙ) Q .+= semfiml.commutator * Q - ∇Σ = P * (imply.∇S + Q * imply.∇A) + ∇Σ = P * (implied.∇S + Q * implied.∇A) - ∇μ = imply.F⨉I_A⁻¹ * imply.∇M + kron((imply.I_A⁻¹ * imply.M)', imply.F⨉I_A⁻¹) * imply.∇A + ∇μ = implied.F⨉I_A⁻¹ * implied.∇M + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A mul!(G, ∇Σ', JΣ) # actually transposed mul!(G, ∇μ', Jμ, -1, 1) @@ -198,7 +198,7 @@ function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) model, ) end - return ∇F_fiml_outer!(G, JΣ, Jμ, imply(model), model, semfiml) + return ∇F_fiml_outer!(G, JΣ, Jμ, implied(model), model, semfiml) end function prepare_SemFIML!(semfiml, model) @@ -212,8 +212,8 @@ function prepare_SemFIML!(semfiml, model) end function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) - Σ = imply(model).Σ - μ = imply(model).μ + Σ = implied(model).Σ + μ = implied(model).μ data = observed(model) @inbounds @views for (i, pat) in enumerate(data.patterns) fiml.inverses[i] .= Σ[pat.measured_mask, pat.measured_mask] @@ -230,7 +230,7 @@ function batch_cholesky!(semfiml, model) end function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, imply(model).Σ) + copyto!(semfiml.imp_inv, implied(model).Σ) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index e81d27de7..d14af648c 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -58,14 +58,14 @@ end ############################################################################################ ############################################################################################ -### Symbolic Imply Types +### Symbolic Implied Types function evaluate!( objective, gradient, hessian, semml::SemML, - implied::SemImplySymbolic, + implied::SemImpliedSymbolic, model::AbstractSemSingle, par, ) @@ -132,7 +132,7 @@ function evaluate!( end ############################################################################################ -### Non-Symbolic Imply Types +### Non-Symbolic Implied Types function evaluate!( objective, @@ -144,7 +144,7 @@ function evaluate!( par, ) if !isnothing(hessian) - error("hessian of ML + non-symbolic imply type is not available") + error("hessian of ML + non-symbolic implied type is not available") end Σ = implied.Σ diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 9702a9cf4..0fe2c9b3c 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -104,7 +104,7 @@ function evaluate!( gradient, hessian, semwls::SemWLS, - implied::SemImplySymbolic, + implied::SemImpliedSymbolic, model::AbstractSemSingle, par, ) diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 6ec59ec39..02f637270 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -8,18 +8,18 @@ Ridge regularization. # Constructor - SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, imply = nothing, kwargs...) + SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, implied = nothing, kwargs...) # Arguments - `α_ridge`: hyperparameter for penalty term - `which_ridge::Vector`: Vector of parameter labels (Symbols) or indices that indicate which parameters should be regularized. - `nparams::Int`: number of parameters of the model -- `imply::SemImply`: imply part of the model +- `implied::SemImplied`: implied part of the model - `parameter_type`: type of the parameters # Examples ```julia -my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, imply = my_imply) +my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, implied = my_implied) ``` # Interfaces @@ -48,18 +48,18 @@ function SemRidge(; which_ridge, nparams, parameter_type = Float64, - imply = nothing, + implied = nothing, kwargs..., ) if eltype(which_ridge) <: Symbol - if isnothing(imply) + if isnothing(implied) throw( ArgumentError( - "When referring to parameters by label, `imply = ...` has to be specified", + "When referring to parameters by label, `implied = ...` has to be specified", ), ) else - par2ind = Dict(par => ind for (ind, par) in enumerate(params(imply))) + par2ind = Dict(par => ind for (ind, par) in enumerate(params(implied))) which_ridge = getindex.(Ref(par2ind), which_ridge) end end diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index f07b572aa..5b430e29e 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -23,28 +23,28 @@ is_hessian_required(::EvaluationTargets{<:Any, <:Any, H}) where {H} = H (targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) -# dispatch on SemImply +# dispatch on SemImplied evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = - evaluate!(objective, gradient, hessian, loss, imply(model), model, params) + evaluate!(objective, gradient, hessian, loss, implied(model), model, params) # fallback method -function evaluate!(obj, grad, hess, loss::SemLossFunction, imply::SemImply, model, params) - isnothing(obj) || (obj = objective(loss, imply, model, params)) - isnothing(grad) || copyto!(grad, gradient(loss, imply, model, params)) - isnothing(hess) || copyto!(hess, hessian(loss, imply, model, params)) +function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, model, params) + isnothing(obj) || (obj = objective(loss, implied, model, params)) + isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params)) + isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params)) return obj end # fallback methods -objective(f::SemLossFunction, imply::SemImply, model, params) = objective(f, model, params) -gradient(f::SemLossFunction, imply::SemImply, model, params) = gradient(f, model, params) -hessian(f::SemLossFunction, imply::SemImply, model, params) = hessian(f, model, params) - -# fallback method for SemImply that calls update_xxx!() methods -function update!(targets::EvaluationTargets, imply::SemImply, model, params) - is_objective_required(targets) && update_objective!(imply, model, params) - is_gradient_required(targets) && update_gradient!(imply, model, params) - is_hessian_required(targets) && update_hessian!(imply, model, params) +objective(f::SemLossFunction, implied::SemImplied, model, params) = objective(f, model, params) +gradient(f::SemLossFunction, implied::SemImplied, model, params) = gradient(f, model, params) +hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params) + +# fallback method for SemImplied that calls update_xxx!() methods +function update!(targets::EvaluationTargets, implied::SemImplied, model, params) + is_objective_required(targets) && update_objective!(implied, model, params) + is_gradient_required(targets) && update_gradient!(implied, model, params) + is_hessian_required(targets) && update_hessian!(implied, model, params) end # guess objective type @@ -72,8 +72,8 @@ objective_zero(objective, gradient, hessian) = function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params) targets = EvaluationTargets(objective, gradient, hessian) - # update imply state, its gradient and hessian (if required) - update!(targets, imply(model), model, params) + # update implied state, its gradient and hessian (if required) + update!(targets, implied(model), model, params) return evaluate!( !isnothing(objective) ? zero(objective) : nothing, gradient, @@ -90,8 +90,8 @@ end function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) function obj(p) - # recalculate imply state for p - update!(EvaluationTargets{true, false, false}(), imply(model), model, p) + # recalculate implied state for p + update!(EvaluationTargets{true, false, false}(), implied(model), model, p) evaluate!( objective_zero(objective, gradient, hessian), nothing, @@ -165,7 +165,7 @@ Returns the objective value at `params`. The model object can be modified. # Implementation -To implement a new `SemImply` or `SemLossFunction` subtype, you need to add a method for +To implement a new `SemImplied` or `SemLossFunction` subtype, you need to add a method for objective!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you need to add a method for @@ -179,7 +179,7 @@ function objective! end Writes the gradient value at `params` to `gradient`. # Implementation -To implement a new `SemImply` or `SemLossFunction` type, you can add a method for +To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for gradient!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for @@ -193,7 +193,7 @@ function gradient! end Writes the hessian value at `params` to `hessian`. # Implementation -To implement a new `SemImply` or `SemLossFunction` type, you can add a method for +To implement a new `SemImplied` or `SemLossFunction` type, you can add a method for hessian!(newtype::MyNewType, params, model::AbstractSemSingle) To implement a new `AbstractSem` subtype, you can add a method for diff --git a/src/optimizer/abstract.jl b/src/optimizer/abstract.jl index c6669aa12..68bcc04ad 100644 --- a/src/optimizer/abstract.jl +++ b/src/optimizer/abstract.jl @@ -110,7 +110,7 @@ function prepare_param_bounds( default::Number, variance_default::Number, ) where {BOUND} - varparams = Set(variance_params(model.imply.ram_matrices)) + varparams = Set(variance_params(model.implied.ram_matrices)) res = [ begin def = in(p, varparams) ? variance_default : default diff --git a/src/types.jl b/src/types.jl index cfe916d9e..e802e057a 100644 --- a/src/types.jl +++ b/src/types.jl @@ -4,17 +4,17 @@ "Most abstract supertype for all SEMs" abstract type AbstractSem end -"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `imply`, `loss`" +"Supertype for all single SEMs, e.g. SEMs that have at least the fields `observed`, `implied`, `loss`" abstract type AbstractSemSingle{O, I, L} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end -"Meanstructure trait for `SemImply` subtypes" +"Meanstructure trait for `SemImplied` subtypes" abstract type MeanStruct end -"Indicates that `SemImply` subtype supports mean structure" +"Indicates that `SemImplied` subtype supports mean structure" struct HasMeanStruct <: MeanStruct end -"Indicates that `SemImply` subtype does not support mean structure" +"Indicates that `SemImplied` subtype does not support mean structure" struct NoMeanStruct <: MeanStruct end # default implementation @@ -24,7 +24,7 @@ MeanStruct(::Type{T}) where {T} = MeanStruct(semobj) = MeanStruct(typeof(semobj)) -"Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" +"Hessian Evaluation trait for `SemImplied` and `SemLossFunction` subtypes" abstract type HessianEval end struct ApproxHessian <: HessianEval end struct ExactHessian <: HessianEval end @@ -105,36 +105,36 @@ If you have a special kind of data, e.g. ordinal data, you should implement a su abstract type SemObserved end """ -Supertype of all objects that can serve as the imply field of a SEM. +Supertype of all objects that can serve as the implied field of a SEM. Computed model-implied values that should be compared with the observed data to find parameter estimates, e. g. the model implied covariance or mean. -If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. +If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImplied. """ -abstract type SemImply end +abstract type SemImplied end -"Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." -abstract type SemImplySymbolic <: SemImply end +"Subtype of SemImplied for all objects that can serve as the implied field of a SEM and use some form of symbolic precomputation." +abstract type SemImpliedSymbolic <: SemImplied end """ - Sem(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) + Sem(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) Constructor for the basic `Sem` type. -All additional kwargs are passed down to the constructors for the observed, imply, and loss fields. +All additional kwargs are passed down to the constructors for the observed, implied, and loss fields. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. -- `imply`: object of subtype `SemImply` or a constructor. +- `implied`: object of subtype `SemImplied` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). +- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). """ -mutable struct Sem{O <: SemObserved, I <: SemImply, L <: SemLoss} <: +mutable struct Sem{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: AbstractSemSingle{O, I, L} observed::O - imply::I + implied::I loss::L end @@ -142,25 +142,25 @@ end # automatic differentiation ############################################################################################ """ - SemFiniteDiff(;observed = SemObservedData, imply = RAM, loss = SemML, kwargs...) + SemFiniteDiff(;observed = SemObservedData, implied = RAM, loss = SemML, kwargs...) A wrapper around [`Sem`](@ref) that substitutes dedicated evaluation of gradient and hessian with finite difference approximation. # Arguments - `observed`: object of subtype `SemObserved` or a constructor. -- `imply`: object of subtype `SemImply` or a constructor. +- `implied`: object of subtype `SemImplied` or a constructor. - `loss`: object of subtype `SemLossFunction`s or constructor; or a tuple of such. Returns a Sem with fields - `observed::SemObserved`: Stores observed data, sample statistics, etc. See also [`SemObserved`](@ref). -- `imply::SemImply`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImply`](@ref). +- `implied::SemImplied`: Computes model implied statistics, like Σ, μ, etc. See also [`SemImplied`](@ref). - `loss::SemLoss`: Computes the objective and gradient of a sum of loss functions. See also [`SemLoss`](@ref). """ -struct SemFiniteDiff{O <: SemObserved, I <: SemImply, L <: SemLoss} <: +struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: AbstractSemSingle{O, I, L} observed::O - imply::I + implied::I loss::L end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 6991dd479..c97c9fb8e 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -4,11 +4,11 @@ const SEM = StructuralEquationModels # ML estimation ############################################################################################ -model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) -model_g2 = Sem(specification = specification_g2, data = dat_g2, imply = RAM) +model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) -@test SEM.params(model_g1.imply.ram_matrices) == SEM.params(model_g2.imply.ram_matrices) +@test SEM.params(model_g1.implied.ram_matrices) == SEM.params(model_g2.implied.ram_matrices) # test the different constructors model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) @@ -94,9 +94,9 @@ specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) specification_g1_s = specification_s[:Pasteur] specification_g2_s = specification_s[:Grant_White] -model_g1 = Sem(specification = specification_g1_s, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1_s, data = dat_g1, implied = RAMSymbolic) -model_g2 = Sem(specification = specification_g2_s, data = dat_g2, imply = RAM) +model_g2 = Sem(specification = specification_g2_s, data = dat_g2, implied = RAM) model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) @@ -145,7 +145,7 @@ end end @testset "sorted | LowerTriangular A" begin - @test imply(model_ml_multigroup.sems[2]).A isa LowerTriangular + @test implied(model_ml_multigroup.sems[2]).A isa LowerTriangular end ############################################################################################ @@ -165,7 +165,7 @@ end using LinearAlgebra: isposdef, logdet, tr, inv function SEM.objective(ml::UserSemML, model::AbstractSem, params) - Σ = imply(model).Σ + Σ = implied(model).Σ Σₒ = SEM.obs_cov(observed(model)) if !isposdef(Σ) return Inf @@ -175,12 +175,12 @@ function SEM.objective(ml::UserSemML, model::AbstractSem, params) end # models -model_g1 = Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic) +model_g1 = Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic) model_g2 = SemFiniteDiff( specification = specification_g2, data = dat_g2, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = UserSemML(), ) @@ -207,10 +207,10 @@ end ############################################################################################ model_ls_g1 = - Sem(specification = specification_g1, data = dat_g1, imply = RAMSymbolic, loss = SemWLS) + Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic, loss = SemWLS) model_ls_g2 = - Sem(specification = specification_g2, data = dat_g2, imply = RAMSymbolic, loss = SemWLS) + Sem(specification = specification_g2, data = dat_g2, implied = RAMSymbolic, loss = SemWLS) model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) @@ -260,7 +260,7 @@ if !isnothing(specification_miss_g1) observed = SemObservedMissing, loss = SemFIML, data = dat_miss_g1, - imply = RAM, + implied = RAM, optimizer = SemOptimizerEmpty(), meanstructure = true, ) @@ -270,7 +270,7 @@ if !isnothing(specification_miss_g1) observed = SemObservedMissing, loss = SemFIML, data = dat_miss_g2, - imply = RAM, + implied = RAM, optimizer = SemOptimizerEmpty(), meanstructure = true, ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 5e5244f91..c99115032 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -5,10 +5,10 @@ # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = spec, data = dat) -# imply -imply_ram = RAM(specification = spec) +# implied +implied_ram = RAM(specification = spec) -imply_ram_sym = RAMSymbolic(specification = spec) +implied_ram_sym = RAMSymbolic(specification = spec) # loss functions --------------------------------------------------------------------------- ml = SemML(observed = observed) @@ -29,18 +29,18 @@ optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml) +model_ml = Sem(observed, implied_ram, loss_ml) model_ls_sym = Sem(observed, RAMSymbolic(specification = spec, vech = true), loss_wls) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) -model_ridge = Sem(observed, imply_ram, SemLoss(ml, ridge)) +model_ridge = Sem(observed, implied_ram, SemLoss(ml, ridge)) -model_constant = Sem(observed, imply_ram, SemLoss(ml, constant)) +model_constant = Sem(observed, implied_ram, SemLoss(ml, constant)) model_ml_weighted = - Sem(observed, imply_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) + Sem(observed, implied_ram, SemLoss(ml; loss_weights = [nsamples(model_ml)])) ############################################################################################ ### test gradients @@ -158,13 +158,13 @@ if opt_engine == :Optim ), ) - imply_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) + implied_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) - imply_sym_hessian = RAMSymbolic(specification = spec, hessian = true) + implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true) - model_ls = Sem(observed, imply_sym_hessian_vech, loss_wls) + model_ls = Sem(observed, implied_sym_hessian_vech, loss_wls) - model_ml = Sem(observed, imply_sym_hessian, loss_ml) + model_ml = Sem(observed, implied_sym_hessian, loss_ml) @testset "ml_hessians" begin test_hessian(model_ml, start_test; atol = 1e-4) @@ -199,10 +199,10 @@ end # observed --------------------------------------------------------------------------------- observed = SemObservedData(specification = spec_mean, data = dat, meanstructure = true) -# imply -imply_ram = RAM(specification = spec_mean, meanstructure = true) +# implied +implied_ram = RAM(specification = spec_mean, meanstructure = true) -imply_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) +implied_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) # loss functions --------------------------------------------------------------------------- ml = SemML(observed = observed, meanstructure = true) @@ -218,7 +218,7 @@ loss_wls = SemLoss(wls) optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- -model_ml = Sem(observed, imply_ram, loss_ml) +model_ml = Sem(observed, implied_ram, loss_ml) model_ls = Sem( observed, @@ -226,7 +226,7 @@ model_ls = Sem( loss_wls, ) -model_ml_sym = Sem(observed, imply_ram_sym, loss_ml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_ml) ############################################################################################ ### test gradients @@ -314,9 +314,9 @@ fiml = SemFIML(observed = observed, specification = spec_mean) loss_fiml = SemLoss(fiml) -model_ml = Sem(observed, imply_ram, loss_fiml) +model_ml = Sem(observed, implied_ram, loss_fiml) -model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml) +model_ml_sym = Sem(observed, implied_ram_sym, loss_fiml) ############################################################################################ ### test gradients diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index cba86aef0..1d18ffed4 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -8,7 +8,7 @@ using Random, NLopt semoptimizer = SemOptimizer(engine = opt_engine) model_ml = Sem(specification = spec, data = dat) -@test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) +@test SEM.params(model_ml.implied.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( specification = spec, @@ -18,9 +18,9 @@ model_ml_cov = Sem( nsamples = 75, ) -model_ls_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic, loss = SemWLS) +model_ls_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic, loss = SemWLS) -model_ml_sym = Sem(specification = spec, data = dat, imply = RAMSymbolic) +model_ml_sym = Sem(specification = spec, data = dat, implied = RAMSymbolic) model_ridge = Sem( specification = spec, @@ -199,7 +199,7 @@ if opt_engine == :Optim model_ls = Sem( specification = spec, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, hessian = true, algorithm = Newton(; @@ -211,7 +211,7 @@ if opt_engine == :Optim model_ml = Sem( specification = spec, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, hessian = true, algorithm = Newton(), ) @@ -251,7 +251,7 @@ end model_ls = Sem( specification = spec_mean, data = dat, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemWLS, meanstructure = true, ) @@ -269,7 +269,7 @@ model_ml_cov = Sem( ) model_ml_sym = - Sem(specification = spec_mean, data = dat, imply = RAMSymbolic, meanstructure = true) + Sem(specification = spec_mean, data = dat, implied = RAMSymbolic, meanstructure = true) ############################################################################################ ### test gradients @@ -405,7 +405,7 @@ model_ml_sym = Sem( specification = spec_mean, data = dat_missing, observed = SemObservedMissing, - imply = RAMSymbolic, + implied = RAMSymbolic, loss = SemFIML, meanstructure = true, ) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 4b968bc49..6899fe7a7 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -53,11 +53,11 @@ start = [ repeat([0.5], 4) ] -imply_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) +implied_ml = RAMSymbolic(; specification = ram_matrices, start_val = start) -imply_ml.Σ_function(imply_ml.Σ, true_val) +implied_ml.Σ_function(implied_ml.Σ, true_val) -true_dist = MultivariateNormal(imply_ml.Σ) +true_dist = MultivariateNormal(implied_ml.Σ) Random.seed!(1234) x = transpose(rand(true_dist, 100_000)) @@ -65,7 +65,7 @@ semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(; observed = semobserved, nparams = length(start))) -model_ml = Sem(semobserved, imply_ml, loss_ml) +model_ml = Sem(semobserved, implied_ml, loss_ml) objective!(model_ml, true_val) optimizer = SemOptimizerOptim( diff --git a/test/unit_tests/model.jl b/test/unit_tests/model.jl index bf44091d2..7ed190c22 100644 --- a/test/unit_tests/model.jl +++ b/test/unit_tests/model.jl @@ -46,25 +46,25 @@ function test_params_api(semobj, spec::SemSpecification) @test @inferred(params(semobj)) == params(spec) end -@testset "Sem(imply=$implytype, loss=$losstype)" for implytype in (RAM, RAMSymbolic), +@testset "Sem(implied=$impliedtype, loss=$losstype)" for impliedtype in (RAM, RAMSymbolic), losstype in (SemML, SemWLS) model = Sem( specification = ram_matrices, observed = obs, - imply = implytype, + implied = impliedtype, loss = losstype, ) @test model isa Sem - @test @inferred(imply(model)) isa implytype + @test @inferred(implied(model)) isa impliedtype @test @inferred(observed(model)) isa SemObserved test_vars_api(model, ram_matrices) test_params_api(model, ram_matrices) - test_vars_api(imply(model), ram_matrices) - test_params_api(imply(model), ram_matrices) + test_vars_api(implied(model), ram_matrices) + test_params_api(implied(model), ram_matrices) @test @inferred(loss(model)) isa SemLoss semloss = loss(model).functions[1] diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index f5bc38ae0..0908a6497 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -7,7 +7,7 @@ sort_vars!(partable) model_ml_sorted = Sem(specification = partable, data = dat) @testset "graph sorting" begin - @test model_ml_sorted.imply.I_A isa LowerTriangular + @test model_ml_sorted.implied.I_A isa LowerTriangular end @testset "ml_solution_sorted" begin From 39aee3d4d87d14a0763db4dcd922bbe599bc83ef Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 22 Dec 2024 19:45:32 -0800 Subject: [PATCH 51/71] imply -> implied: file renames --- docs/make.jl | 2 +- docs/src/developer/{imply.md => implied.md} | 0 src/StructuralEquationModels.jl | 8 ++++---- src/{imply => implied}/RAM/generic.jl | 0 src/{imply => implied}/RAM/symbolic.jl | 0 src/{imply => implied}/abstract.jl | 0 src/{imply => implied}/empty.jl | 0 7 files changed, 5 insertions(+), 5 deletions(-) rename docs/src/developer/{imply.md => implied.md} (100%) rename src/{imply => implied}/RAM/generic.jl (100%) rename src/{imply => implied}/RAM/symbolic.jl (100%) rename src/{imply => implied}/abstract.jl (100%) rename src/{imply => implied}/empty.jl (100%) diff --git a/docs/make.jl b/docs/make.jl index 4a55d55ce..4542cf48f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -32,7 +32,7 @@ makedocs( "Developer documentation" => [ "Extending the package" => "developer/extending.md", "Custom loss functions" => "developer/loss.md", - "Custom imply types" => "developer/imply.md", + "Custom implied types" => "developer/implied.md", "Custom optimizer types" => "developer/optimizer.md", "Custom observed types" => "developer/observed.md", "Custom model types" => "developer/sem.md", diff --git a/docs/src/developer/imply.md b/docs/src/developer/implied.md similarity index 100% rename from docs/src/developer/imply.md rename to docs/src/developer/implied.md diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a9a5af0d7..8bcf7a78d 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -50,10 +50,10 @@ include("observed/EM.jl") include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") # implied -include("imply/abstract.jl") -include("imply/RAM/symbolic.jl") -include("imply/RAM/generic.jl") -include("imply/empty.jl") +include("implied/abstract.jl") +include("implied/RAM/symbolic.jl") +include("implied/RAM/generic.jl") +include("implied/empty.jl") # loss include("loss/ML/ML.jl") include("loss/ML/FIML.jl") diff --git a/src/imply/RAM/generic.jl b/src/implied/RAM/generic.jl similarity index 100% rename from src/imply/RAM/generic.jl rename to src/implied/RAM/generic.jl diff --git a/src/imply/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl similarity index 100% rename from src/imply/RAM/symbolic.jl rename to src/implied/RAM/symbolic.jl diff --git a/src/imply/abstract.jl b/src/implied/abstract.jl similarity index 100% rename from src/imply/abstract.jl rename to src/implied/abstract.jl diff --git a/src/imply/empty.jl b/src/implied/empty.jl similarity index 100% rename from src/imply/empty.jl rename to src/implied/empty.jl From 1fe165be02fc810197359cb2dff99e234464a7b7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 12:09:04 +0100 Subject: [PATCH 52/71] close #158 --- docs/src/tutorials/collection/multigroup.md | 2 +- docs/src/tutorials/constraints/constraints.md | 4 ++-- docs/src/tutorials/first_model.md | 6 +++--- docs/src/tutorials/inspection/inspection.md | 12 ++++++------ .../src/tutorials/regularization/regularization.md | 4 ++-- src/StructuralEquationModels.jl | 2 +- src/frontend/fit/summary.jl | 14 +++++++------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index 5ee88e936..d0fc71796 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -83,7 +83,7 @@ We now fit the model and inspect the parameter estimates: ```@example mg; ansicolor = true solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) -sem_summary(partable) +details(partable) ``` Other things you can query about your fitted model (fit measures, standard errors, etc.) are described in the section [Model inspection](@ref) and work the same way for multigroup models. \ No newline at end of file diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index a67ad7372..ffd83d4e0 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -52,7 +52,7 @@ model_fit = sem_fit(model) update_estimate!(partable, model_fit) -sem_summary(partable) +details(partable) ``` ### Define the constraints @@ -165,7 +165,7 @@ update_partable!( solution(model_fit_constrained), ) -sem_summary(partable) +details(partable) ``` As we can see, the constrained solution is very close to the original solution (compare the columns estimate and estimate_constr), with the difference that the constrained parameters fulfill their constraints. diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index 7568a5917..a285e29df 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -119,10 +119,10 @@ and compute fit measures as fit_measures(model_fit) ``` -We can also get a bit more information about the fitted model via the `sem_summary()` function: +We can also get a bit more information about the fitted model via the `details()` function: ```@example high_level; ansicolor = true -sem_summary(model_fit) +details(model_fit) ``` To investigate the parameter estimates, we can update our `partable` object to contain the new estimates: @@ -134,7 +134,7 @@ update_estimate!(partable, model_fit) and investigate the solution with ```@example high_level; ansicolor = true -sem_summary(partable) +details(partable) ``` Congratulations, you fitted and inspected your very first model! diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index 88caf5812..faab8f8ed 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -53,10 +53,10 @@ model_fit = sem_fit(model) you end up with an object of type [`SemFit`](@ref). -You can get some more information about it by using the `sem_summary` function: +You can get some more information about it by using the `details` function: ```@example colored; ansicolor = true -sem_summary(model_fit) +details(model_fit) ``` To compute fit measures, we use @@ -73,12 +73,12 @@ AIC(model_fit) A list of available [Fit measures](@ref) is at the end of this page. -To inspect the parameter estimates, we can update a `ParameterTable` object and call `sem_summary` on it: +To inspect the parameter estimates, we can update a `ParameterTable` object and call `details` on it: ```@example colored; ansicolor = true; output = false update_estimate!(partable, model_fit) -sem_summary(partable) +details(partable) ``` We can also update the `ParameterTable` object with other information via [`update_partable!`](@ref). For example, if we want to compare hessian-based and bootstrap-based standard errors, we may write @@ -90,7 +90,7 @@ se_he = se_hessian(model_fit) update_partable!(partable, :se_hessian, params(model_fit), se_he) update_partable!(partable, :se_bootstrap, params(model_fit), se_bs) -sem_summary(partable) +details(partable) ``` ## Export results @@ -106,7 +106,7 @@ parameters_df = DataFrame(partable) # API - model inspection ```@docs -sem_summary +details update_estimate! update_partable! ``` diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 4aaff1d0a..02d3b3bac 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -148,7 +148,7 @@ update_estimate!(partable, fit) update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lasso)) -sem_summary(partable) +details(partable) ``` ## Second example - mixed l1 and l0 regularization @@ -182,5 +182,5 @@ Let's again compare the different results: ```@example reg update_partable!(partable, :estimate_mixed, params(fit_mixed), solution(fit_mixed)) -sem_summary(partable) +details(partable) ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..b0ca407ff 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -131,7 +131,7 @@ export AbstractSem, SemFit, minimum, solution, - sem_summary, + details, objective!, gradient!, hessian!, diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index e6026e5f4..d9b137a58 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -1,4 +1,4 @@ -function sem_summary( +function details( sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, @@ -45,7 +45,7 @@ function sem_summary( print("\n") end -function sem_summary( +function details( partable::ParameterTable; color = :light_cyan, secondary_color = :light_yellow, @@ -250,7 +250,7 @@ function sem_summary( end -function sem_summary( +function details( partable::EnsembleParameterTable; color = :light_cyan, secondary_color = :light_yellow, @@ -291,7 +291,7 @@ function sem_summary( print("\n") printstyled(rpad(" Group: $k", 78), reverse = true) print("\n") - sem_summary( + details( partable.tables[k]; color = color, secondary_color = secondary_color, @@ -333,9 +333,9 @@ function Base.findall(fun::Function, partable::ParameterTable) end """ - (1) sem_summary(sem_fit::SemFit; show_fitmeasures = false) + (1) details(sem_fit::SemFit; show_fitmeasures = false) - (2) sem_summary(partable::AbstractParameterTable; ...) + (2) details(partable::AbstractParameterTable; ...) Print information about (1) a fitted SEM or (2) a parameter table to stdout. @@ -347,4 +347,4 @@ Print information about (1) a fitted SEM or (2) a parameter table to stdout. - `show_variables = true` - `show_columns = nothing`: columns names to include in the output e.g.`[:from, :to, :estimate]`) """ -function sem_summary end +function details end From e051d714fb631531b0ae7359ab5f76ad9871a766 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 12:20:31 +0100 Subject: [PATCH 53/71] close #232 --- docs/src/developer/loss.md | 2 +- docs/src/performance/simulation.md | 12 +++++------ src/StructuralEquationModels.jl | 2 +- src/additional_functions/simulation.jl | 20 +++++++++---------- src/frontend/fit/standard_errors/bootstrap.jl | 4 ++-- .../political_democracy/constructor.jl | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index e1137dbf1..96d1ba566 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -195,7 +195,7 @@ end ### Update observed data If you are planing a simulation study where you have to fit the **same model** to many **different datasets**, it is computationally beneficial to not build the whole model completely new everytime you change your data. -Therefore, we provide a function to update the data of your model, `swap_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy: +Therefore, we provide a function to update the data of your model, `replace_observed(model(semfit); data = new_data)`. However, we can not know beforehand in what way your loss function depends on the specific datasets. The solution is to provide a method for `update_observed`. Since `Ridge` does not depend on the data at all, this is quite easy: ```julia import StructuralEquationModels: update_observed diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index b8a5081fe..e46be64a7 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -7,12 +7,12 @@ ## Swap observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. -We therefore provide the function `swap_observed` to change the `observed` part of a model, +We therefore provide the function `replace_observed` to change the `observed` part of a model, without necessarily reconstructing the other parts. For the [A first model](@ref), you would use it as -```@setup swap_observed +```@setup replace_observed using StructuralEquationModels observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] @@ -49,7 +49,7 @@ partable = ParameterTable( ) ``` -```@example swap_observed +```@example replace_observed data = example_data("political_democracy") data_1 = data[1:30, :] @@ -61,7 +61,7 @@ model = Sem( data = data_1 ) -model_updated = swap_observed(model; data = data_2, specification = partable) +model_updated = replace_observed(model; data = data_2, specification = partable) ``` !!! danger "Thread safety" @@ -76,7 +76,7 @@ model_updated = swap_observed(model; data = data_2, specification = partable) If you are building your models by parts, you can also update each part seperately with the function `update_observed`. For example, -```@example swap_observed +```@example replace_observed new_observed = SemObservedData(;data = data_2, specification = partable) @@ -88,6 +88,6 @@ new_optimizer = update_observed(my_optimizer, new_observed) ## API ```@docs -swap_observed +replace_observed update_observed ``` \ No newline at end of file diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index a6677a4ed..ed49704f4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -177,7 +177,7 @@ export AbstractSem, se_hessian, se_bootstrap, example_data, - swap_observed, + replace_observed, update_observed, @StenoGraph, →, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 0b2626b15..e33b4f2fe 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -1,7 +1,7 @@ """ - (1) swap_observed(model::AbstractSemSingle; kwargs...) + (1) replace_observed(model::AbstractSemSingle; kwargs...) - (2) swap_observed(model::AbstractSemSingle, observed; kwargs...) + (2) replace_observed(model::AbstractSemSingle, observed; kwargs...) Return a new model with swaped observed part. @@ -13,7 +13,7 @@ Return a new model with swaped observed part. # Examples See the online documentation on [Swap observed data](@ref). """ -function swap_observed end +function replace_observed end """ update_observed(to_update, observed::SemObserved; kwargs...) @@ -34,15 +34,15 @@ function update_observed end ############################################################################################ # use the same observed type as before -swap_observed(model::AbstractSemSingle; kwargs...) = - swap_observed(model, typeof(observed(model)).name.wrapper; kwargs...) +replace_observed(model::AbstractSemSingle; kwargs...) = + replace_observed(model, typeof(observed(model)).name.wrapper; kwargs...) # construct a new observed type -swap_observed(model::AbstractSemSingle, observed_type; kwargs...) = - swap_observed(model, observed_type(; kwargs...); kwargs...) +replace_observed(model::AbstractSemSingle, observed_type; kwargs...) = + replace_observed(model, observed_type(; kwargs...); kwargs...) -swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = - swap_observed( +replace_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = + replace_observed( model, observed(model), imply(model), @@ -51,7 +51,7 @@ swap_observed(model::AbstractSemSingle, new_observed::SemObserved; kwargs...) = kwargs..., ) -function swap_observed( +function replace_observed( model::AbstractSemSingle, old_observed, imply, diff --git a/src/frontend/fit/standard_errors/bootstrap.jl b/src/frontend/fit/standard_errors/bootstrap.jl index 9695e4cb3..e8d840d0c 100644 --- a/src/frontend/fit/standard_errors/bootstrap.jl +++ b/src/frontend/fit/standard_errors/bootstrap.jl @@ -7,7 +7,7 @@ Only works for single models. # Arguments - `n_boot`: number of boostrap samples - `data`: data to sample from. Only needed if different than the data from `sem_fit` -- `kwargs...`: passed down to `swap_observed` +- `kwargs...`: passed down to `replace_observed` """ function se_bootstrap( semfit::SemFit; @@ -42,7 +42,7 @@ function se_bootstrap( for _ in 1:n_boot sample_data = bootstrap_sample(data) - new_model = swap_observed( + new_model = replace_observed( model(semfit); data = sample_data, specification = specification, diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index cba86aef0..acab3b8f4 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -169,13 +169,13 @@ end Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) # simulate data - model_ml_new = swap_observed( + model_ml_new = replace_observed( model_ml, data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, ) - model_ml_sym_new = swap_observed( + model_ml_sym_new = replace_observed( model_ml_sym, data = rand(model_ml_sym, params, 1_000_000), specification = spec, @@ -366,14 +366,14 @@ end Random.seed!(83472834) colnames = Symbol.(names(example_data("political_democracy"))) # simulate data - model_ml_new = swap_observed( + model_ml_new = replace_observed( model_ml, data = rand(model_ml, params, 1_000_000), specification = spec, obs_colnames = colnames, meanstructure = true, ) - model_ml_sym_new = swap_observed( + model_ml_sym_new = replace_observed( model_ml_sym, data = rand(model_ml_sym, params, 1_000_000), specification = spec, From 8b0f8805328bb89e71a56a9f31b00747f1a2a055 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst <34346372+Maximilian-Stefan-Ernst@users.noreply.github.com> Date: Sun, 2 Feb 2025 13:37:59 +0100 Subject: [PATCH 54/71] Update ext/SEMProximalOptExt/ProximalAlgorithms.jl --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index f82c2b005..94fcad247 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -61,7 +61,6 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) return obj, grad end -#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) mutable struct ProximalResult result::Any From 8c703d6bd6add573cf6ebe3f2471909415c06de6 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Sun, 2 Feb 2025 15:00:13 +0100 Subject: [PATCH 55/71] suppress uninformative warnings during package testing --- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 1 - src/frontend/fit/summary.jl | 19 +++++----- .../specification/EnsembleParameterTable.jl | 8 ++--- src/frontend/specification/ParameterTable.jl | 2 +- src/frontend/specification/RAMMatrices.jl | 32 ++++++++++------- src/implied/abstract.jl | 5 +-- src/loss/ML/FIML.jl | 4 ++- src/objective_gradient_hessian.jl | 16 +++++++-- test/Project.toml | 1 + test/examples/multigroup/build_models.jl | 23 +++++++----- test/examples/multigroup/multigroup.jl | 8 ++--- test/examples/political_democracy/by_parts.jl | 7 ++-- .../political_democracy/constructor.jl | 4 +-- .../political_democracy.jl | 36 ++----------------- test/examples/proximal/ridge.jl | 4 +-- test/unit_tests/data_input_formats.jl | 6 ++-- 16 files changed, 83 insertions(+), 93 deletions(-) diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index 94fcad247..eceff0dc3 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -61,7 +61,6 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params) return obj, grad end - mutable struct ProximalResult result::Any end diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index d9b137a58..70bf6816c 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -1,9 +1,4 @@ -function details( - sem_fit::SemFit; - show_fitmeasures = false, - color = :light_cyan, - digits = 2, -) +function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan, digits = 2) print("\n") println("Fitted Structural Equation Model") print("\n") @@ -51,7 +46,7 @@ function details( secondary_color = :light_yellow, digits = 2, show_variables = true, - show_columns = nothing + show_columns = nothing, ) if show_variables print("\n") @@ -150,7 +145,8 @@ function details( check_round(partable.columns[c][regression_indices]; digits = digits) for c in regression_columns ) - regression_columns[2] = regression_columns[2] == :relation ? Symbol("") : regression_columns[2] + regression_columns[2] = + regression_columns[2] == :relation ? Symbol("") : regression_columns[2] print("\n") pretty_table( @@ -222,7 +218,8 @@ function details( printstyled("Means: \n"; color = color) if isnothing(show_columns) - sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = + [:from, :relation, :to, :estimate, :param, :value_fixed, :start] mean_columns = sort_partially(sorted_columns, columns) else mean_columns = copy(show_columns) @@ -256,7 +253,7 @@ function details( secondary_color = :light_yellow, digits = 2, show_variables = true, - show_columns = nothing + show_columns = nothing, ) if show_variables print("\n") @@ -297,7 +294,7 @@ function details( secondary_color = secondary_color, digits = digits, show_variables = false, - show_columns = show_columns + show_columns = show_columns, ) end diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index b1c8fb8e6..d5ac7e51b 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -19,7 +19,7 @@ EnsembleParameterTable(::Nothing; params::Union{Nothing, Vector{Symbol}} = nothi ) # convert pairs to dict -EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = +EnsembleParameterTable(ps::Pair{K, V}...; params = nothing) where {K, V} = EnsembleParameterTable(Dict(ps...); params = params) # dictionary of SEM specifications @@ -148,8 +148,6 @@ end ############################################################################################ function Base.:(==)(p1::EnsembleParameterTable, p2::EnsembleParameterTable) - out = - (p1.tables == p2.tables) && - (p1.params == p2.params) + out = (p1.tables == p2.tables) && (p1.params == p2.params) return out -end \ No newline at end of file +end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 07c24e46e..c5ad010b3 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -128,7 +128,7 @@ end # Equality -------------------------------------------------------------------------------- function Base.:(==)(p1::ParameterTable, p2::ParameterTable) - out = + out = (p1.columns == p2.columns) && (p1.observed_vars == p2.observed_vars) && (p1.latent_vars == p2.latent_vars) && diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 0c5722f57..43fd87945 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -133,8 +133,10 @@ function RAMMatrices( @assert length(partable.sorted_vars) == nvars(partable) vars_sorted = copy(partable.sorted_vars) else - vars_sorted = [partable.observed_vars - partable.latent_vars] + vars_sorted = [ + partable.observed_vars + partable.latent_vars + ] end # indices of the vars (A/S/M rows or columns) @@ -216,13 +218,20 @@ function RAMMatrices( sort!(M_consts, by = first) end - return RAMMatrices(ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), - ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), - sparse(1:n_observed, - [vars_index[var] for var in partable.observed_vars], - ones(T, n_observed), n_observed, n_vars), - !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, - params, vars_sorted) + return RAMMatrices( + ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), + ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), + sparse( + 1:n_observed, + [vars_index[var] for var in partable.observed_vars], + ones(T, n_observed), + n_observed, + n_vars, + ), + !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, + params, + vars_sorted, + ) end Base.convert( @@ -360,10 +369,7 @@ function append_rows!( arr_ix = arr_ixs[arr.linear_indices[j]] skip_symmetric && (arr_ix ∈ visited_indices) && continue - push!( - partable, - partable_row(par, arr_ix, arr_name, varnames, free = true), - ) + push!(partable, partable_row(par, arr_ix, arr_name, varnames, free = true)) if skip_symmetric # mark index and its symmetric as visited push!(visited_indices, arr_ix) diff --git a/src/implied/abstract.jl b/src/implied/abstract.jl index 05b0e2449..99bb4d68d 100644 --- a/src/implied/abstract.jl +++ b/src/implied/abstract.jl @@ -25,8 +25,9 @@ function check_acyclic(A::AbstractMatrix; verbose::Bool = false) # check if non-triangular matrix is acyclic acyclic = isone(det(I - A)) if acyclic - verbose && @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = - 1 + verbose && + @info "The matrix is acyclic. Reordering variables in the model to make the A matrix either Upper or Lower Triangular can significantly improve performance.\n" maxlog = + 1 end return A end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 2c398090a..0ef542f70 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -161,7 +161,9 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, implied, model, semfiml) ∇Σ = P * (implied.∇S + Q * implied.∇A) - ∇μ = implied.F⨉I_A⁻¹ * implied.∇M + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A + ∇μ = + implied.F⨉I_A⁻¹ * implied.∇M + + kron((implied.I_A⁻¹ * implied.M)', implied.F⨉I_A⁻¹) * implied.∇A mul!(G, ∇Σ', JΣ) # actually transposed mul!(G, ∇μ', Jμ, -1, 1) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 5b430e29e..4aafe4235 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -28,7 +28,15 @@ evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSe evaluate!(objective, gradient, hessian, loss, implied(model), model, params) # fallback method -function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, model, params) +function evaluate!( + obj, + grad, + hess, + loss::SemLossFunction, + implied::SemImplied, + model, + params, +) isnothing(obj) || (obj = objective(loss, implied, model, params)) isnothing(grad) || copyto!(grad, gradient(loss, implied, model, params)) isnothing(hess) || copyto!(hess, hessian(loss, implied, model, params)) @@ -36,8 +44,10 @@ function evaluate!(obj, grad, hess, loss::SemLossFunction, implied::SemImplied, end # fallback methods -objective(f::SemLossFunction, implied::SemImplied, model, params) = objective(f, model, params) -gradient(f::SemLossFunction, implied::SemImplied, model, params) = gradient(f, model, params) +objective(f::SemLossFunction, implied::SemImplied, model, params) = + objective(f, model, params) +gradient(f::SemLossFunction, implied::SemImplied, model, params) = + gradient(f, model, params) hessian(f::SemLossFunction, implied::SemImplied, model, params) = hessian(f, model, params) # fallback method for SemImplied that calls update_xxx!() methods diff --git a/test/Project.toml b/test/Project.toml index 59db0b155..3cf1e50e3 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -15,4 +15,5 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index c97c9fb8e..2f5135176 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -17,10 +17,9 @@ model_ml_multigroup2 = SemEnsemble( data = dat, column = :school, groups = [:Pasteur, :Grant_White], - loss = SemML + loss = SemML, ) - # gradients @testset "ml_gradients_multigroup" begin test_gradient(model_ml_multigroup, start_test; atol = 1e-9) @@ -206,11 +205,19 @@ end # GLS estimation ############################################################################################ -model_ls_g1 = - Sem(specification = specification_g1, data = dat_g1, implied = RAMSymbolic, loss = SemWLS) +model_ls_g1 = Sem( + specification = specification_g1, + data = dat_g1, + implied = RAMSymbolic, + loss = SemWLS, +) -model_ls_g2 = - Sem(specification = specification_g2, data = dat_g2, implied = RAMSymbolic, loss = SemWLS) +model_ls_g2 = Sem( + specification = specification_g2, + data = dat_g2, + implied = RAMSymbolic, + loss = SemWLS, +) model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) @@ -239,7 +246,7 @@ end atol = 1e-5, ) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -283,7 +290,7 @@ if !isnothing(specification_miss_g1) groups = [:Pasteur, :Grant_White], loss = SemFIML, observed = SemObservedMissing, - meanstructure = true + meanstructure = true, ) ############################################################################################ diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index caaa5c3f7..7dd871ac2 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, FiniteDiff +using StructuralEquationModels, Test, FiniteDiff, Suppressor using LinearAlgebra: diagind, LowerTriangular const SEM = StructuralEquationModels @@ -71,10 +71,8 @@ specification_g2 = RAMMatrices(; vars = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed], ) -partable = EnsembleParameterTable( - :Pasteur => specification_g1, - :Grant_White => specification_g2 -) +partable = + EnsembleParameterTable(:Pasteur => specification_g1, :Grant_White => specification_g2) specification_miss_g1 = nothing specification_miss_g2 = nothing diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index c99115032..88f98ded2 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -133,7 +133,7 @@ end ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -158,7 +158,8 @@ if opt_engine == :Optim ), ) - implied_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) + implied_sym_hessian_vech = + RAMSymbolic(specification = spec, vech = true, hessian = true) implied_sym_hessian = RAMSymbolic(specification = spec, hessian = true) @@ -294,7 +295,7 @@ end ) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_se_hessian!(partable_mean, solution_ls) + @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 67538afa7..bbeb0c648 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -141,7 +141,7 @@ end ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_se_hessian!(partable, solution_ls) + @suppress update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; @@ -337,7 +337,7 @@ end ) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_se_hessian!(partable_mean, solution_ls) + @suppress update_se_hessian!(partable_mean, solution_ls) test_estimates( partable_mean, solution_lav[:parameter_estimates_ls_mean]; diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index a2e5089bb..9d026fb28 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, FiniteDiff +using StructuralEquationModels, Test, Suppressor, FiniteDiff SEM = StructuralEquationModels @@ -78,22 +78,7 @@ spec = RAMMatrices(; S = S, F = F, params = x, - vars = [ - :x1, - :x2, - :x3, - :y1, - :y2, - :y3, - :y4, - :y5, - :y6, - :y7, - :y8, - :ind60, - :dem60, - :dem65, - ], + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable = ParameterTable(spec) @@ -110,22 +95,7 @@ spec_mean = RAMMatrices(; F = F, M = M, params = [SEM.params(spec); Symbol.("x", string.(32:38))], - vars = [ - :x1, - :x2, - :x3, - :y1, - :y2, - :y3, - :y4, - :y5, - :y6, - :y7, - :y8, - :ind60, - :dem60, - :dem65, - ], + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65], ) partable_mean = ParameterTable(spec_mean) diff --git a/test/examples/proximal/ridge.jl b/test/examples/proximal/ridge.jl index 16a318a12..8c0a1df7a 100644 --- a/test/examples/proximal/ridge.jl +++ b/test/examples/proximal/ridge.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators +using StructuralEquationModels, Test, ProximalAlgorithms, ProximalOperators, Suppressor # load data dat = example_data("political_democracy") @@ -54,7 +54,7 @@ solution_ridge = sem_fit(model_ridge) model_prox = Sem(specification = partable, data = dat, loss = SemML) -solution_prox = sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) +solution_prox = @suppress sem_fit(model_prox, engine = :Proximal, operator_g = SqrNormL2(λ)) @testset "ridge_solution" begin @test isapprox(solution_prox.solution, solution_ridge.solution; rtol = 1e-4) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index cc72673a6..183b067f5 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,4 +1,4 @@ -using StructuralEquationModels, Test, Statistics +using StructuralEquationModels, Test, Statistics, Suppressor ### model specification -------------------------------------------------------------------- @@ -189,7 +189,7 @@ end ) # spec takes precedence in obs_vars order - observed_spec = SemObservedData( + observed_spec = @suppress SemObservedData( specification = spec, data = shuffle_dat, observed_vars = shuffle_names, @@ -451,7 +451,7 @@ end # SemObservedCovariance ) # spec takes precedence in obs_vars order - observed_spec = SemObservedMissing( + observed_spec = @suppress SemObservedMissing( specification = spec, observed_vars = shuffle_names, data = shuffle_dat_missing, From ba9d2c92571e20ee5b03f43a27dbc7828271848d Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Mon, 3 Feb 2025 17:48:33 +0100 Subject: [PATCH 56/71] turn simplification of symbolic terms by default off --- src/implied/RAM/symbolic.jl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/implied/RAM/symbolic.jl b/src/implied/RAM/symbolic.jl index 07acef019..44ad4949d 100644 --- a/src/implied/RAM/symbolic.jl +++ b/src/implied/RAM/symbolic.jl @@ -93,6 +93,7 @@ function RAMSymbolic(; specification::SemSpecification, loss_types = nothing, vech = false, + simplify_symbolics = false, gradient = true, hessian = false, meanstructure = false, @@ -116,7 +117,7 @@ function RAMSymbolic(; I_A⁻¹ = neumann_series(A) # Σ - Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech) + Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech, simplify = simplify_symbolics) #print(Symbolics.build_function(Σ_symbolic)[2]) Σ_function = Symbolics.build_function(Σ_symbolic, par, expression = Val{false})[2] Σ = zeros(size(Σ_symbolic)) @@ -157,7 +158,7 @@ function RAMSymbolic(; # μ if meanstructure MS = HasMeanStruct - μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F) + μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F; simplify = simplify_symbolics) μ_function = Symbolics.build_function(μ_symbolic, par, expression = Val{false})[2] μ = zeros(size(μ_symbolic)) if gradient @@ -235,23 +236,26 @@ end ############################################################################################ # expected covariations of observed vars -function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false) +function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false, simplify = false) Σ = F * I_A⁻¹ * S * permutedims(I_A⁻¹) * permutedims(F) Σ = Array(Σ) vech && (Σ = Σ[tril(trues(size(F, 1), size(F, 1)))]) - # Σ = Symbolics.simplify.(Σ) - Threads.@threads for i in eachindex(Σ) - Σ[i] = Symbolics.simplify(Σ[i]) + if simplify + Threads.@threads for i in eachindex(Σ) + Σ[i] = Symbolics.simplify(Σ[i]) + end end return Σ end # expected means of observed vars -function eval_μ_symbolic(M, I_A⁻¹, F) +function eval_μ_symbolic(M, I_A⁻¹, F; simplify = false) μ = F * I_A⁻¹ * M μ = Array(μ) - Threads.@threads for i in eachindex(μ) - μ[i] = Symbolics.simplify(μ[i]) + if simplify + Threads.@threads for i in eachindex(μ) + μ[i] = Symbolics.simplify(μ[i]) + end end return μ end From cd6413b60bdd81341a687268e4847bf4555453d7 Mon Sep 17 00:00:00 2001 From: Aaron Peikert Date: Mon, 3 Feb 2025 19:30:22 +0100 Subject: [PATCH 57/71] new version of StenoGraph results in fewer deprication notices --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5937930d3..d55346aca 100644 --- a/Project.toml +++ b/Project.toml @@ -25,7 +25,7 @@ SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] julia = "1.9, 1.10" -StenoGraphs = "0.2, 0.3" +StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" DataFrames = "1" Distributions = "0.25" FiniteDiff = "2" From f0df6538f0220f964cbf51772698c317a0b4cf86 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:43:31 +0100 Subject: [PATCH 58/71] fix exporting structs from package extensions --- ext/SEMNLOptExt/NLopt.jl | 67 --------------------- ext/SEMProximalOptExt/ProximalAlgorithms.jl | 25 -------- src/StructuralEquationModels.jl | 8 ++- src/package_extensions/SEMNLOptExt.jl | 64 ++++++++++++++++++++ src/package_extensions/SEMProximalOptExt.jl | 21 +++++++ 5 files changed, 92 insertions(+), 93 deletions(-) create mode 100644 src/package_extensions/SEMNLOptExt.jl create mode 100644 src/package_extensions/SEMProximalOptExt.jl diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index 959380292..ff868afc2 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,70 +1,3 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `NLopt.jl` as the optimization backend. - -# Constructor - - SemOptimizerNLopt(; - algorithm = :LD_LBFGS, - options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), - kwargs...) - -# Arguments -- `algorithm`: optimization algorithm. -- `options::Dict{Symbol, Any}`: options for the optimization algorithm -- `local_algorithm`: local optimization algorithm -- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm -- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints -- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints - -# Example -```julia -my_optimizer = SemOptimizerNLopt() - -# constrained optimization with augmented lagrangian -my_constrained_optimizer = SemOptimizerNLopt(; - algorithm = :AUGLAG, - local_algorithm = :LD_LBFGS, - local_options = Dict(:ftol_rel => 1e-6), - inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), -) -``` - -# Usage -All algorithms and options from the NLopt library are available, for more information see -the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, -see [Constrained optimization](@ref) in our online documentation. - -# Extended help - -## Interfaces -- `algorithm(::SemOptimizerNLopt)` -- `local_algorithm(::SemOptimizerNLopt)` -- `options(::SemOptimizerNLopt)` -- `local_options(::SemOptimizerNLopt)` -- `equality_constraints(::SemOptimizerNLopt)` -- `inequality_constraints(::SemOptimizerNLopt)` - -## Implementation - -Subtype of `SemOptimizer`. -""" -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} - algorithm::A - local_algorithm::A2 - options::B - local_options::B2 - equality_constraints::C - inequality_constraints::C -end - Base.@kwdef struct NLoptConstraint f::Any tol = 0.0 diff --git a/ext/SEMProximalOptExt/ProximalAlgorithms.jl b/ext/SEMProximalOptExt/ProximalAlgorithms.jl index eceff0dc3..2f1775e85 100644 --- a/ext/SEMProximalOptExt/ProximalAlgorithms.jl +++ b/ext/SEMProximalOptExt/ProximalAlgorithms.jl @@ -1,28 +1,3 @@ -############################################################################################ -### Types -############################################################################################ -""" -Connects to `ProximalAlgorithms.jl` as the optimization backend. - -# Constructor - - SemOptimizerProximal(; - algorithm = ProximalAlgorithms.PANOC(), - operator_g, - operator_h = nothing, - kwargs..., - -# Arguments -- `algorithm`: optimization algorithm. -- `operator_g`: gradient of the objective function -- `operator_h`: optional hessian of the objective function -""" -mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} - algorithm::A - operator_g::B - operator_h::C -end - SEM.SemOptimizer{:Proximal}(args...; kwargs...) = SemOptimizerProximal(args...; kwargs...) SemOptimizerProximal(; diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 7c8923dc8..af2cd4bfe 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -82,6 +82,10 @@ include("frontend/fit/fitmeasures/fit_measures.jl") # standard errors include("frontend/fit/standard_errors/hessian.jl") include("frontend/fit/standard_errors/bootstrap.jl") +# extensions +include("package_extensions/SEMNLOptExt.jl") +include("package_extensions/SEMProximalOptExt.jl") + export AbstractSem, AbstractSemSingle, @@ -183,5 +187,7 @@ export AbstractSem, →, ←, ↔, - ⇔ + ⇔, + SemOptimizerNLopt, + SemOptimizerProximal end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl new file mode 100644 index 000000000..5d6d090c4 --- /dev/null +++ b/src/package_extensions/SEMNLOptExt.jl @@ -0,0 +1,64 @@ +""" +Connects to `NLopt.jl` as the optimization backend. +Only usable if `NLopt.jl` is loaded in the current Julia session! + +# Constructor + + SemOptimizerNLopt(; + algorithm = :LD_LBFGS, + options = Dict{Symbol, Any}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), + kwargs...) + +# Arguments +- `algorithm`: optimization algorithm. +- `options::Dict{Symbol, Any}`: options for the optimization algorithm +- `local_algorithm`: local optimization algorithm +- `local_options::Dict{Symbol, Any}`: options for the local optimization algorithm +- `equality_constraints::Vector{NLoptConstraint}`: vector of equality constraints +- `inequality_constraints::Vector{NLoptConstraint}`: vector of inequality constraints + +# Example +```julia +my_optimizer = SemOptimizerNLopt() + +# constrained optimization with augmented lagrangian +my_constrained_optimizer = SemOptimizerNLopt(; + algorithm = :AUGLAG, + local_algorithm = :LD_LBFGS, + local_options = Dict(:ftol_rel => 1e-6), + inequality_constraints = NLoptConstraint(;f = my_constraint, tol = 0.0), +) +``` + +# Usage +All algorithms and options from the NLopt library are available, for more information see +the NLopt.jl package and the NLopt online documentation. +For information on how to use inequality and equality constraints, +see [Constrained optimization](@ref) in our online documentation. + +# Extended help + +## Interfaces +- `algorithm(::SemOptimizerNLopt)` +- `local_algorithm(::SemOptimizerNLopt)` +- `options(::SemOptimizerNLopt)` +- `local_options(::SemOptimizerNLopt)` +- `equality_constraints(::SemOptimizerNLopt)` +- `inequality_constraints(::SemOptimizerNLopt)` + +## Implementation + +Subtype of `SemOptimizer`. +""" +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} + algorithm::A + local_algorithm::A2 + options::B + local_options::B2 + equality_constraints::C + inequality_constraints::C +end \ No newline at end of file diff --git a/src/package_extensions/SEMProximalOptExt.jl b/src/package_extensions/SEMProximalOptExt.jl new file mode 100644 index 000000000..e8b256704 --- /dev/null +++ b/src/package_extensions/SEMProximalOptExt.jl @@ -0,0 +1,21 @@ +""" +Connects to `ProximalAlgorithms.jl` as the optimization backend. + +# Constructor + + SemOptimizerProximal(; + algorithm = ProximalAlgorithms.PANOC(), + operator_g, + operator_h = nothing, + kwargs..., + +# Arguments +- `algorithm`: optimization algorithm. +- `operator_g`: gradient of the objective function +- `operator_h`: optional hessian of the objective function +""" +mutable struct SemOptimizerProximal{A, B, C} <: SemOptimizer{:Proximal} + algorithm::A + operator_g::B + operator_h::C +end \ No newline at end of file From 81a4bd9839df01e9f487b9aa13e3df107856114a Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:50:16 +0100 Subject: [PATCH 59/71] fix NLopt extension --- ext/SEMNLOptExt/NLopt.jl | 5 ----- ext/SEMNLOptExt/SEMNLOptExt.jl | 3 +-- src/StructuralEquationModels.jl | 1 + src/package_extensions/SEMNLOptExt.jl | 5 +++++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ext/SEMNLOptExt/NLopt.jl b/ext/SEMNLOptExt/NLopt.jl index ff868afc2..a614c501b 100644 --- a/ext/SEMNLOptExt/NLopt.jl +++ b/ext/SEMNLOptExt/NLopt.jl @@ -1,8 +1,3 @@ -Base.@kwdef struct NLoptConstraint - f::Any - tol = 0.0 -end - Base.convert( ::Type{NLoptConstraint}, tuple::NamedTuple{(:f, :tol), Tuple{F, T}}, diff --git a/ext/SEMNLOptExt/SEMNLOptExt.jl b/ext/SEMNLOptExt/SEMNLOptExt.jl index a159f6dc8..c79fc2b86 100644 --- a/ext/SEMNLOptExt/SEMNLOptExt.jl +++ b/ext/SEMNLOptExt/SEMNLOptExt.jl @@ -1,11 +1,10 @@ module SEMNLOptExt using StructuralEquationModels, NLopt +using StructuralEquationModels: SemOptimizerNLopt, NLoptConstraint SEM = StructuralEquationModels -export SemOptimizerNLopt, NLoptConstraint - include("NLopt.jl") end diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index af2cd4bfe..5d6b23ef4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -189,5 +189,6 @@ export AbstractSem, ↔, ⇔, SemOptimizerNLopt, + NLoptConstraint, SemOptimizerProximal end diff --git a/src/package_extensions/SEMNLOptExt.jl b/src/package_extensions/SEMNLOptExt.jl index 5d6d090c4..7eae2f268 100644 --- a/src/package_extensions/SEMNLOptExt.jl +++ b/src/package_extensions/SEMNLOptExt.jl @@ -61,4 +61,9 @@ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} local_options::B2 equality_constraints::C inequality_constraints::C +end + +Base.@kwdef struct NLoptConstraint + f::Any + tol = 0.0 end \ No newline at end of file From 9729819b86f375e4663de1fe9ec9c38d4932f580 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:54:12 +0100 Subject: [PATCH 60/71] fix Proximal extension --- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 156311367..0db21462d 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -2,8 +2,7 @@ module SEMProximalOptExt using StructuralEquationModels using ProximalAlgorithms - -export SemOptimizerProximal +using StructuralEquationModels: SemOptimizerProximal SEM = StructuralEquationModels From 127da26bd7e24007d2ab136429d4d024364d0329 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 11:13:33 +0100 Subject: [PATCH 61/71] fix printing --- .../regularization/regularization.md | 33 +++++-------------- ext/SEMProximalOptExt/SEMProximalOptExt.jl | 2 +- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index 02d3b3bac..f9d19b176 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -5,40 +5,23 @@ For ridge regularization, you can simply use `SemRidge` as an additional loss function (for example, a model with the loss functions `SemML` and `SemRidge` corresponds to ridge-regularized maximum likelihood estimation). -For lasso, elastic net and (far) beyond, we provide the `ProximalSEM` package. You can install it and load it alongside `StructuralEquationModels`: +For lasso, elastic net and (far) beyond, you can load the `ProximalAlgorithms.jl` and `ProximalOperators.jl` packages alongside `StructuralEquationModels`: ```@setup reg -import Pkg -Pkg.add(url = "https://github.com/StructuralEquationModels/ProximalSEM.jl") - -using StructuralEquationModels, ProximalSEM -``` - -```julia -import Pkg -Pkg.add(url = "https://github.com/StructuralEquationModels/ProximalSEM.jl") - -using StructuralEquationModels, ProximalSEM -``` - -!!! warning "ProximalSEM is still WIP" - The ProximalSEM package does not have any releases yet, and is not well tested - until the first release, use at your own risk and expect interfaces to change without prior notice. - -Additionally, you need to install and load `ProximalOperators.jl`: - -```@setup reg -using ProximalOperators +using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ``` ```julia +using Pkg +Pkg.add("ProximalAlgorithms") Pkg.add("ProximalOperators") -using ProximalOperators +using StructuralEquationModels, ProximalAlgorithms, ProximalOperators ``` ## `SemOptimizerProximal` -`ProximalSEM` provides a new "building block" for the optimizer part of a model, called `SemOptimizerProximal`. +To estimate regularized models, we provide a "building block" for the optimizer part, called `SemOptimizerProximal`. It connects our package to the [`ProximalAlgorithms.jl`](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) optimization backend, providing so-called proximal optimization algorithms. Those can handle, amongst other things, various forms of regularization. @@ -102,7 +85,9 @@ model = Sem( We labeled the covariances between the items because we want to regularize those: ```@example reg -ind = param_indices([:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68], model) +ind = getindex.( + [param_indices(model)], + [:cov_15, :cov_24, :cov_26, :cov_37, :cov_48, :cov_68]) ``` In the following, we fit the same model with lasso regularization of those covariances. diff --git a/ext/SEMProximalOptExt/SEMProximalOptExt.jl b/ext/SEMProximalOptExt/SEMProximalOptExt.jl index 0db21462d..192944fef 100644 --- a/ext/SEMProximalOptExt/SEMProximalOptExt.jl +++ b/ext/SEMProximalOptExt/SEMProximalOptExt.jl @@ -2,7 +2,7 @@ module SEMProximalOptExt using StructuralEquationModels using ProximalAlgorithms -using StructuralEquationModels: SemOptimizerProximal +using StructuralEquationModels: SemOptimizerProximal, print_type_name, print_field_types SEM = StructuralEquationModels From f67d48cb76a822b5bbcd6b48a71ae2a9f1fab420 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 11:28:29 +0100 Subject: [PATCH 62/71] fix regularization docs --- .../regularization/regularization.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/src/tutorials/regularization/regularization.md b/docs/src/tutorials/regularization/regularization.md index f9d19b176..37e42975a 100644 --- a/docs/src/tutorials/regularization/regularization.md +++ b/docs/src/tutorials/regularization/regularization.md @@ -112,8 +112,7 @@ optimizer_lasso = SemOptimizerProximal( model_lasso = Sem( specification = partable, - data = data, - optimizer = optimizer_lasso + data = data ) ``` @@ -121,7 +120,7 @@ Let's fit the regularized model ```@example reg -fit_lasso = sem_fit(model_lasso) +fit_lasso = sem_fit(optimizer_lasso, model_lasso) ``` and compare the solution to unregularizted estimates: @@ -136,6 +135,12 @@ update_partable!(partable, :estimate_lasso, params(fit_lasso), solution(fit_lass details(partable) ``` +Instead of explicitely defining a `SemOptimizerProximal` object, you can also pass `engine = :Proximal` and additional keyword arguments to `sem_fit`: + +```@example reg +fit = sem_fit(model; engine = :Proximal, operator_g = NormL1(λ)) +``` + ## Second example - mixed l1 and l0 regularization You can choose to penalize different parameters with different types of regularization functions. @@ -150,16 +155,14 @@ To define a sup of separable proximal operators (i.e. no parameter is penalized we can use [`SlicedSeparableSum`](https://juliafirstorder.github.io/ProximalOperators.jl/stable/calculus/#ProximalOperators.SlicedSeparableSum) from the `ProximalOperators` package: ```@example reg -prox_operator = SlicedSeparableSum((NormL1(0.02), NormL0(20.0), NormL0(0.0)), ([ind], [12:22], [vcat(1:11, 23:25)])) +prox_operator = SlicedSeparableSum((NormL0(20.0), NormL1(0.02), NormL0(0.0)), ([ind], [9:11], [vcat(1:8, 12:25)])) model_mixed = Sem( specification = partable, - data = data, - optimizer = SemOptimizerProximal, - operator_g = prox_operator + data = data, ) -fit_mixed = sem_fit(model_mixed) +fit_mixed = sem_fit(model_mixed; engine = :Proximal, operator_g = prox_operator) ``` Let's again compare the different results: From e9dbb62a24dec7e5eeb2a014f88474d141fba646 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 10:27:41 +0100 Subject: [PATCH 63/71] start reworking docs --- docs/Project.toml | 3 + docs/src/assets/concept.svg | 155 +++++++---------- docs/src/assets/concept_typed.svg | 156 +++++++----------- docs/src/index.md | 2 +- docs/src/tutorials/backends/nlopt.md | 3 + docs/src/tutorials/concept.md | 12 +- .../tutorials/construction/build_by_parts.md | 18 +- .../construction/outer_constructor.md | 25 +-- docs/src/tutorials/first_model.md | 20 +-- .../specification/graph_interface.md | 22 +-- .../tutorials/specification/ram_matrices.md | 4 +- .../tutorials/specification/specification.md | 8 +- 12 files changed, 191 insertions(+), 237 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 9da7f0ab4..2daded98f 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,4 +1,7 @@ [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" +ProximalSEM = "3652f839-8142-48b2-a17c-985bd14407c5" diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index 2a7a0b42a..138463b67 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1,197 +1,166 @@ + id="defs23" /> - - - + inkscape:deskcolor="#d1d1d1"> + + + id="path3" /> + id="path4" /> + id="path5" /> + id="path6" /> + id="path7" /> + id="path8" /> + id="path9" /> + id="path10" /> + id="path11" /> + id="path12" /> + id="path13" /> + id="path14" /> - - - + id="path15" /> + id="path16" /> + id="path17" /> + id="path18" /> + id="path19" /> + id="path20" /> + id="path21" /> + id="path22" /> + id="path23" /> diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index 8281300f8..032adc9b8 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1,197 +1,169 @@ + id="defs23" /> - - - + inkscape:deskcolor="#d1d1d1"> + + + id="path3" /> + id="path4" /> + id="path5" /> + id="path6" /> + id="path7" /> + id="path8" /> + id="path9" /> + id="path10" /> + id="path11" /> + id="path12" /> + id="path13" /> + id="path14" /> - - - + id="path15" /> + id="path16" /> + id="path17" /> + id="path18" /> + id="path19" /> + id="path20" /> + id="path21" /> + id="path22" /> + id="path23" /> diff --git a/docs/src/index.md b/docs/src/index.md index 8b2d6999e..add69459e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -32,7 +32,7 @@ For examples of how to use the package, see the Tutorials. Models you can fit out of the box include - Linear SEM that can be specified in RAM notation - ML, GLS and FIML estimation -- Ridge Regularization +- Ridge/Lasso/... Regularization - Multigroup SEM - Sums of arbitrary loss functions (everything the optimizer can handle) diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index d4c5fdf8f..f861e174e 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -1,6 +1,7 @@ # Using NLopt.jl [`SemOptimizerNLopt`](@ref) implements the connection to `NLopt.jl`. +It is only available if the `NLopt` package is loaded alongside `StructuralEquationModel.jl` in the running Julia session. It takes a bunch of arguments: ```julia @@ -22,6 +23,8 @@ The defaults are LBFGS as the optimization algorithm and the standard options fr We can choose something different: ```julia +using NLopt + my_optimizer = SemOptimizerNLopt(; algorithm = :AUGLAG, options = Dict(:maxeval => 200), diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index b8d094abc..d663d3c2c 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -1,12 +1,13 @@ # Our Concept of a Structural Equation Model -In our package, every Structural Equation Model (`Sem`) consists of four parts: +In our package, every Structural Equation Model (`Sem`) consists of three parts (four, if you count the optimizer): ![SEM concept](../assets/concept.svg) Those parts are interchangable building blocks (like 'Legos'), i.e. there are different pieces available you can choose as the `observed` slot of the model, and stick them together with other pieces that can serve as the `implied` part. -The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix) and the optimizer part connects to the optimization backend (e.g. the type of optimization algorithm used). +The `observed` part is for observed data, the `implied` part is what the model implies about your data (e.g. the model implied covariance matrix), and the loss part compares the observed data and implied properties (e.g. weighted least squares difference between the observed and implied covariance matrix). +The optimizer part is not part of the model itself, but it is needed to fit the model as it connects to the optimization backend (e.g. the type of optimization algorithm used). For example, to build a model for maximum likelihood estimation with the NLopt optimization suite as a backend you would choose `SemML` as a loss function and `SemOptimizerNLopt` as the optimizer. @@ -51,12 +52,12 @@ Available loss functions are ## The optimizer part aka `SemOptimizer` The optimizer part of a model connects to the numerical optimization backend used to fit the model. It can be used to control options like the optimization algorithm, linesearch, stopping criteria, etc. -There are currently two available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, and [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend. -For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref). +There are currently three available backends, [`SemOptimizerOptim`](@ref) connecting to the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) backend, [`SemOptimizerNLopt`](@ref) connecting to the [NLopt.jl](https://github.com/JuliaOpt/NLopt.jl) backend and [`SemOptimizerProximal`](@ref) connecting to [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl). +For more information about the available options see also the tutorials about [Using Optim.jl](@ref) and [Using NLopt.jl](@ref), as well as [Constrained optimization](@ref) and [Regularization](@ref) . # What to do next -You now have an understanding about our representation of structural equation models. +You now have an understanding of our representation of structural equation models. To learn more about how to use the package, you may visit the remaining tutorials. @@ -100,4 +101,5 @@ SemConstant SemOptimizer SemOptimizerOptim SemOptimizerNLopt +SemOptimizerProximal ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 071750a8c..27604d2a1 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -11,8 +11,8 @@ using StructuralEquationModels data = example_data("political_democracy") -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -27,8 +27,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -40,8 +40,8 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` Now, we construct the different parts: @@ -59,9 +59,11 @@ ml = SemML(observed = observed) loss_ml = SemLoss(ml) # optimizer ------------------------------------------------------------------------------------- -optimizer = SemOptimizerOptim() +optimizer = SemOptimizerOptim(algorithm = BFGS()) # model ------------------------------------------------------------------------------------ -model_ml = Sem(observed, implied_ram, loss_ml, optimizer) +model_ml = Sem(observed, implied_ram, loss_ml) + +sem_fit(optimizer, model_ml) ``` \ No newline at end of file diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 0979f684a..7de3d9e61 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -35,7 +35,7 @@ model = Sem( ) ``` -For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the NLopt backend, write +For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the Optim backend, write ```julia model = Sem( @@ -43,7 +43,7 @@ model = Sem( data = data, implied = RAMSymbolic, loss = SemWLS, - optimizer = SemOptimizerNLopt + optimizer = SemOptimizerOptim ) ``` @@ -92,25 +92,29 @@ help>SemObservedMissing For observed data with missing values. Constructor - ≡≡≡≡≡≡≡≡≡≡≡≡≡ + ≡≡≡≡≡≡≡≡≡≡≡ SemObservedMissing(; - specification, data, - obs_colnames = nothing, + observed_vars = nothing, + specification = nothing, kwargs...) Arguments - ≡≡≡≡≡≡≡≡≡≡≡ + ≡≡≡≡≡≡≡≡≡ - • specification: either a RAMMatrices or ParameterTable object (1) + • specification: optional SEM model specification + (SemSpecification) • data: observed data - • obs_colnames::Vector{Symbol}: column names of the data (if the object passed as data does not have column names, i.e. is not a data frame) + • observed_vars::Vector{Symbol}: column names of the data (if + the object passed as data does not have column names, i.e. is + not a data frame) + + ──────────────────────────────────────────────────────────────────────── - ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── -Extended help is available with `??` +Extended help is available with `??SemObservedMissing` ``` ## Optimize loss functions without analytic gradient @@ -118,7 +122,6 @@ Extended help is available with `??` For loss functions without analytic gradients, it is possible to use finite difference approximation or automatic differentiation. All loss functions provided in the package do have analytic gradients (and some even hessians or approximations thereof), so there is no need do use this feature if you are only working with them. However, if you implement your own loss function, you do not have to provide analytic gradients. -This page is a about finite difference approximation. For information about how to use automatic differentiation, see the documentation of the [AutoDiffSEM](https://github.com/StructuralEquationModels/AutoDiffSEM) package. To use finite difference approximation, you may construct your model just as before, but swap the `Sem` constructor for `SemFiniteDiff`. For example diff --git a/docs/src/tutorials/first_model.md b/docs/src/tutorials/first_model.md index a285e29df..5b7284649 100644 --- a/docs/src/tutorials/first_model.md +++ b/docs/src/tutorials/first_model.md @@ -15,8 +15,8 @@ using StructuralEquationModels We then first define the graph of our model in a syntax which is similar to the R-package `lavaan`: ```@setup high_level -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -31,8 +31,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -44,8 +44,8 @@ end ``` ```julia -observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -latent_vars = [:ind60, :dem60, :dem65] +obs_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] +lat_vars = [:ind60, :dem60, :dem65] graph = @StenoGraph begin @@ -60,8 +60,8 @@ graph = @StenoGraph begin ind60 → dem65 # variances - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ↔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ↔ _(lat_vars) # covariances y1 ↔ y5 @@ -84,8 +84,8 @@ We then use this graph to define a `ParameterTable` object ```@example high_level; ansicolor = true partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` load the example data diff --git a/docs/src/tutorials/specification/graph_interface.md b/docs/src/tutorials/specification/graph_interface.md index 609c844c3..75e1d1b6d 100644 --- a/docs/src/tutorials/specification/graph_interface.md +++ b/docs/src/tutorials/specification/graph_interface.md @@ -12,13 +12,13 @@ end and convert it to a ParameterTable to construct your models: ```julia -observed_vars = ... -latent_vars = ... +obs_vars = ... +lat_vars = ... partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) model = Sem( specification = partable, @@ -66,23 +66,23 @@ As you saw above and in the [A first model](@ref) example, the graph object need ```julia partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) ``` The `ParameterTable` constructor also needs you to specify a vector of observed and latent variables, in the example above this would correspond to ```julia -observed_vars = [:x1 :x2 :x3 :x4 :x5 :x6 :x7 :x8 :x9] -latent_vars = [:ξ₁ :ξ₂ :ξ₃] +obs_vars = [:x1 :x2 :x3 :x4 :x5 :x6 :x7 :x8 :x9] +lat_vars = [:ξ₁ :ξ₂ :ξ₃] ``` The variable names (`:x1`) have to be symbols, the syntax `:something` creates an object of type `Symbol`. But you can also use vectors of symbols inside the graph specification, escaping them with `_(...)`. For example, this graph specification ```julia @StenoGraph begin - _(observed_vars) ↔ _(observed_vars) - _(latent_vars) ⇔ _(latent_vars) + _(obs_vars) ↔ _(obs_vars) + _(lat_vars) ⇔ _(lat_vars) end ``` creates undirected effects coresponding to @@ -95,7 +95,7 @@ Mean parameters are specified as a directed effect from `1` to the respective va ```julia @StenoGraph begin - Symbol("1") → _(observed_vars) + Symbol(1) → _(obs_vars) end ``` diff --git a/docs/src/tutorials/specification/ram_matrices.md b/docs/src/tutorials/specification/ram_matrices.md index 5f0757238..6e01eb38b 100644 --- a/docs/src/tutorials/specification/ram_matrices.md +++ b/docs/src/tutorials/specification/ram_matrices.md @@ -60,7 +60,7 @@ spec = RAMMatrices(; S = S, F = F, params = θ, - colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) model = Sem( @@ -91,7 +91,7 @@ spec = RAMMatrices(; S = S, F = F, params = θ, - colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] + vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) ``` diff --git a/docs/src/tutorials/specification/specification.md b/docs/src/tutorials/specification/specification.md index c426443f4..85bb37c00 100644 --- a/docs/src/tutorials/specification/specification.md +++ b/docs/src/tutorials/specification/specification.md @@ -10,8 +10,8 @@ This leads to the following chart: You can enter model specification at each point, but in general (and especially if you come from `lavaan`), it is the easiest to follow the red arrows: specify a graph object, convert it to a prameter table, and use this parameter table to construct your models ( just like we did in [A first model](@ref)): ```julia -observed_vars = ... -latent_vars = ... +obs_vars = ... +lat_vars = ... graph = @StenoGraph begin ... @@ -19,8 +19,8 @@ end partable = ParameterTable( graph, - latent_vars = latent_vars, - observed_vars = observed_vars) + latent_vars = lat_vars, + observed_vars = obs_vars) model = Sem( specification = partable, From cca249660907e7a264e68014be2aa2accca5c238 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 13:48:21 +0100 Subject: [PATCH 64/71] finish rewriting docs --- docs/src/developer/extending.md | 2 +- docs/src/developer/implied.md | 95 +++++++++++-------- docs/src/developer/loss.md | 80 ++++++---------- docs/src/developer/optimizer.md | 87 ++++++++--------- docs/src/developer/sem.md | 25 +---- docs/src/internals/files.md | 7 +- docs/src/internals/types.md | 4 +- docs/src/performance/simulation.md | 35 +++++-- docs/src/tutorials/backends/nlopt.md | 2 + docs/src/tutorials/backends/optim.md | 4 +- docs/src/tutorials/collection/collection.md | 3 +- docs/src/tutorials/collection/multigroup.md | 4 +- docs/src/tutorials/constraints/constraints.md | 5 +- .../construction/outer_constructor.md | 5 +- docs/src/tutorials/fitting/fitting.md | 24 ++++- docs/src/tutorials/meanstructure.md | 6 +- src/additional_functions/simulation.jl | 4 +- src/frontend/fit/summary.jl | 2 +- src/frontend/specification/ParameterTable.jl | 4 +- src/frontend/specification/RAMMatrices.jl | 10 +- src/implied/empty.jl | 10 +- test/examples/multigroup/build_models.jl | 18 ++-- test/examples/multigroup/multigroup.jl | 6 +- .../political_democracy.jl | 4 +- 24 files changed, 223 insertions(+), 223 deletions(-) diff --git a/docs/src/developer/extending.md b/docs/src/developer/extending.md index 074a8b710..5c3183da4 100644 --- a/docs/src/developer/extending.md +++ b/docs/src/developer/extending.md @@ -1,6 +1,6 @@ # Extending the package -As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of four parts: +As discussed in the section on [Model Construction](@ref), every Structural Equation Model (`Sem`) consists of three (four with the optimizer) parts: ![SEM concept typed](../assets/concept_typed.svg) diff --git a/docs/src/developer/implied.md b/docs/src/developer/implied.md index 403ecfa84..bea824a94 100644 --- a/docs/src/developer/implied.md +++ b/docs/src/developer/implied.md @@ -10,78 +10,89 @@ struct MyImplied <: SemImplied end ``` -and at least a method to compute the objective +and a method to update!: ```julia import StructuralEquationModels: objective! -function objective!(implied::MyImplied, par, model::AbstractSemSingle) - ... - return nothing -end -``` +function update!(targets::EvaluationTargets, implied::MyImplied, model::AbstractSemSingle, params) -This method should compute and store things you want to make available to the loss functions, and returns `nothing`. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `Σ(implied)`. -To make stored computations available to loss functions, simply write a function - for example, for the `RAM` implied type we defined + if is_objective_required(targets) + ... + end -```julia -Σ(implied::RAM) = implied.Σ + if is_gradient_required(targets) + ... + end + if is_hessian_required(targets) + ... + end + +end ``` -Additionally, you can specify methods for `gradient` and `hessian` as well as the combinations described in [Custom loss functions](@ref). +As you can see, `update` gets passed as a first argument `targets`, which is telling us whether the objective value, gradient, and/or hessian are needed. +We can then use the functions `is_..._required` and conditional on what the optimizer needs, we can compute and store things we want to make available to the loss functions. For example, as we have seen in [Second example - maximum likelihood](@ref), the `RAM` implied type computes the model-implied covariance matrix and makes it available via `implied.Σ`. -The last thing nedded to make it work is a method for `nparams` that takes your implied type and returns the number of parameters of the model: -```julia -nparams(implied::MyImplied) = ... -``` Just as described in [Custom loss functions](@ref), you may define a constructor. Typically, this will depend on the `specification = ...` argument that can be a `ParameterTable` or a `RAMMatrices` object. We implement an `ImpliedEmpty` type in our package that does nothing but serving as an `implied` field in case you are using a loss function that does not need any implied type at all. You may use it as a template for defining your own implied type, as it also shows how to handle the specification objects: ```julia -############################################################################ +############################################################################################ ### Types -############################################################################ +############################################################################################ +""" +Empty placeholder for models that don't need an implied part. +(For example, models that only regularize parameters.) -struct ImpliedEmpty{V, V2} <: SemImplied - identifier::V2 - n_par::V -end +# Constructor -############################################################################ -### Constructors -############################################################################ + ImpliedEmpty(;specification, kwargs...) + +# Arguments +- `specification`: either a `RAMMatrices` or `ParameterTable` object + +# Examples +A multigroup model with ridge regularization could be specified as a `SemEnsemble` with one +model per group and an additional model with `ImpliedEmpty` and `SemRidge` for the regularization part. -function ImpliedEmpty(; - specification, - kwargs...) +# Extended help - ram_matrices = RAMMatrices(specification) - identifier = StructuralEquationModels.identifier(ram_matrices) +## Interfaces +- `params(::RAMSymbolic) `-> Vector of parameter labels +- `nparams(::RAMSymbolic)` -> Number of parameters - n_par = length(ram_matrices.parameters) +## Implementation +Subtype of `SemImplied`. +""" +struct ImpliedEmpty{A, B, C} <: SemImplied + hessianeval::A + meanstruct::B + ram_matrices::C +end + +############################################################################################ +### Constructors +############################################################################################ - return ImpliedEmpty(identifier, n_par) +function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) + return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end -############################################################################ +############################################################################################ ### methods -############################################################################ +############################################################################################ -objective!(implied::ImpliedEmpty, par, model) = nothing -gradient!(implied::ImpliedEmpty, par, model) = nothing -hessian!(implied::ImpliedEmpty, par, model) = nothing +update!(targets::EvaluationTargets, implied::ImpliedEmpty, par, model) = nothing -############################################################################ +############################################################################################ ### Recommended methods -############################################################################ - -identifier(implied::ImpliedEmpty) = implied.identifier -n_par(implied::ImpliedEmpty) = implied.n_par +############################################################################################ update_observed(implied::ImpliedEmpty, observed::SemObserved; kwargs...) = implied ``` -As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. Additionally, you should store the `identifier` from the specification object and write a method for `identifier`, as this will make it possible to access parameter indices by label. \ No newline at end of file +As you see, similar to [Custom loss functions](@ref) we implement a method for `update_observed`. \ No newline at end of file diff --git a/docs/src/developer/loss.md b/docs/src/developer/loss.md index 6d709b3be..57a7b485d 100644 --- a/docs/src/developer/loss.md +++ b/docs/src/developer/loss.md @@ -20,17 +20,22 @@ end ``` We store the hyperparameter α and the indices I of the parameters we want to regularize. -Additionaly, we need to define a *method* to compute the objective: +Additionaly, we need to define a *method* of the function `evaluate!` to compute the objective: ```@example loss -import StructuralEquationModels: objective! +import StructuralEquationModels: evaluate! -objective!(ridge::Ridge, par, model::AbstractSemSingle) = ridge.α*sum(par[ridge.I].^2) +evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) = + ridge.α * sum(i -> par[i]^2, ridge.I) ``` +The function `evaluate!` recognizes by the types of the arguments `objective`, `gradient` and `hessian` whether it should compute the objective value, gradient or hessian of the model w.r.t. the parameters. +In this case, `gradient` and `hessian` are of type `Nothing`, signifying that they should not be computed, but only the objective value. + That's all we need to make it work! For example, we can now fit [A first model](@ref) with ridge regularization: We first give some parameters labels to be able to identify them as targets for the regularization: + ```@example loss observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] latent_vars = [:ind60, :dem60, :dem65] @@ -65,7 +70,7 @@ partable = ParameterTable( observed_vars = observed_vars ) -parameter_indices = param_indices([:a, :b, :c], partable) +parameter_indices = getindex.([param_indices(partable)], [:a, :b, :c]) myridge = Ridge(0.01, parameter_indices) model = SemFiniteDiff( @@ -86,15 +91,23 @@ Note that the last argument to the `objective!` method is the whole model. There By far the biggest improvements in performance will result from specifying analytical gradients. We can do this for our example: ```@example loss -import StructuralEquationModels: gradient! - -function gradient!(ridge::Ridge, par, model::AbstractSemSingle) - gradient = zero(par) - gradient[ridge.I] .= 2*ridge.α*par[ridge.I] - return gradient +function evaluate!(objective, gradient, hessian::Nothing, ridge::Ridge, model::AbstractSem, par) + # compute gradient + if !isnothing(gradient) + fill!(gradient, 0) + gradient[ridge.I] .= 2 * ridge.α * par[ridge.I] + end + # compute objective + if !isnothing(objective) + return ridge.α * sum(i -> par[i]^2, ridge.I) + end end ``` +As you can see, in this method definition, both `objective` and `gradient` can be different from `nothing`. +We then check whether to compute the objective value and/or the gradient with `isnothing(objective)`/`isnothing(gradient)`. +This syntax makes it possible to compute objective value and gradient at the same time, which is beneficial when the the objective and gradient share common computations. + Now, instead of specifying a `SemFiniteDiff`, we can use the normal `Sem` constructor: ```@example loss @@ -119,46 +132,7 @@ using BenchmarkTools The exact results of those benchmarks are of course highly depended an your system (processor, RAM, etc.), but you should see that the median computation time with analytical gradients drops to about 5% of the computation without analytical gradients. -Additionally, you may provide analytic hessians by writing a method of the form - -```julia -function hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return hessian -end -``` - -however, this will only matter if you use an optimization algorithm that makes use of the hessians. Our default algorithmn `LBFGS` from the package `Optim.jl` does not use hessians (for example, the `Newton` algorithmn from the same package does). - -To improve performance even more, you can write a method of the form - -```julia -function objective_gradient!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, gradient -end -``` - -This is beneficial when the computation of the objective and gradient share common computations. For example, in maximum likelihood estimation, the model implied covariance matrix has to be inverted to both compute the objective and gradient. Whenever the optimization algorithmn asks for the objective value and gradient at the same point, we call `objective_gradient!` and only have to do the shared computations - in this case the matrix inversion - once. - -If you want to do hessian-based optimization, there are also the following methods: - -```julia -function objective_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, hessian -end - -function gradient_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return gradient, hessian -end - -function objective_gradient_hessian!(ridge::Ridge, par, model::AbstractSemSingle) - ... - return objective, gradient, hessian -end -``` +Additionally, you may provide analytic hessians by writing a respective method for `evaluate!`. However, this will only matter if you use an optimization algorithm that makes use of the hessians. Our default algorithmn `LBFGS` from the package `Optim.jl` does not use hessians (for example, the `Newton` algorithmn from the same package does). ## Convenient @@ -241,11 +215,11 @@ With this information, we write can implement maximum likelihood optimization as struct MaximumLikelihood <: SemLossFunction end using LinearAlgebra -import StructuralEquationModels: Σ, obs_cov, objective! +import StructuralEquationModels: obs_cov, evaluate! -function objective!(semml::MaximumLikelihood, parameters, model::AbstractSem) +function evaluate!(objective::Number, gradient::Nothing, hessian::Nothing, semml::MaximumLikelihood, model::AbstractSem, par) # access the model implied and observed covariance matrices - Σᵢ = Σ(implied(model)) + Σᵢ = implied(model).Σ Σₒ = obs_cov(observed(model)) # compute the objective if isposdef(Symmetric(Σᵢ)) # is the model implied covariance matrix positive definite? diff --git a/docs/src/developer/optimizer.md b/docs/src/developer/optimizer.md index 7480a9d91..82ec594d8 100644 --- a/docs/src/developer/optimizer.md +++ b/docs/src/developer/optimizer.md @@ -1,83 +1,70 @@ # Custom optimizer types The optimizer part of a model connects it to the optimization backend. -The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: +Let's say we want to implement a new optimizer as `SemOptimizerName`. The first part of the implementation is very similar to loss functions, so we just show the implementation of `SemOptimizerOptim` here as a reference: ```julia -############################################################################ +############################################################################################ ### Types and Constructor -############################################################################ - -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer +############################################################################################ +mutable struct SemOptimizerName{A, B} <: SemOptimizer{:Name} algorithm::A options::B end -function SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), - kwargs...) - return SemOptimizerOptim(algorithm, options) -end +SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...) + +SemOptimizerName(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerName(algorithm, options) -############################################################################ +############################################################################################ ### Recommended methods -############################################################################ +############################################################################################ -update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...) = optimizer +update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) = optimizer -############################################################################ +############################################################################################ ### additional methods -############################################################################ +############################################################################################ -algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm -options(optimizer::SemOptimizerOptim) = optimizer.options +algorithm(optimizer::SemOptimizerName) = optimizer.algorithm +options(optimizer::SemOptimizerName) = optimizer.options ``` -Now comes a part that is a little bit more complicated: We need to write methods for `sem_fit`: - -```julia -function sem_fit( - model::AbstractSemSingle{O, I, L, D}; - start_val = start_val, - kwargs...) where {O, I, L, D <: SemOptimizerOptim} - - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - optimization_result = ... - - ... - - return SemFit(minimum, minimizer, start_val, model, optimization_result) -end -``` +Note that your optimizer is a subtype of `SemOptimizer{:Name}`, where you can choose a `:Name` that can later be used as a keyword argument to `sem_fit(engine = :Name)`. +Similarly, `SemOptimizer{:Name}(args...; kwargs...) = SemOptimizerName(args...; kwargs...)` should be defined as well as a constructor that uses only keyword arguments: -The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend). +´´´julia +SemOptimizerName(; + algorithm = LBFGS(), + options = Optim.Options(; f_tol = 1e-10, x_tol = 1.5e-8), + kwargs..., +) = SemOptimizerName(algorithm, options) +´´´ +A method for `update_observed` and additional methods might be usefull, but are not necessary. -If we want our type to also work with `SemEnsemble` models, we also have to provide a method for that: +Now comes the substantive part: We need to provide a method for `sem_fit`: ```julia function sem_fit( - model::SemEnsemble{N, T , V, D, S}; - start_val = start_val, - kwargs...) where {N, T, V, D <: SemOptimizerOptim, S} - - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - + optim::SemOptimizerName, + model::AbstractSem, + start_params::AbstractVector; + kwargs..., +) optimization_result = ... ... - return SemFit(minimum, minimizer, start_val, model, optimization_result) - + return SemFit(minimum, minimizer, start_params, model, optimization_result) end ``` +The method has to return a `SemFit` object that consists of the minimum of the objective at the solution, the minimizer (aka parameter estimates), the starting values, the model and the optimization result (which may be anything you desire for your specific backend). + In addition, you might want to provide methods to access properties of your optimization result: ```julia diff --git a/docs/src/developer/sem.md b/docs/src/developer/sem.md index 0063a85cf..c54ff26af 100644 --- a/docs/src/developer/sem.md +++ b/docs/src/developer/sem.md @@ -1,37 +1,22 @@ # Custom model types -The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L, D}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L, D}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as +The abstract supertype for all models is `AbstractSem`, which has two subtypes, `AbstractSemSingle{O, I, L}` and `AbstractSemCollection`. Currently, there are 2 subtypes of `AbstractSemSingle`: `Sem`, `SemFiniteDiff`. All subtypes of `AbstractSemSingle` should have at least observed, implied, loss and optimizer fields, and share their types (`{O, I, L}`) with the parametric abstract supertype. For example, the `SemFiniteDiff` type is implemented as ```julia -struct SemFiniteDiff{ - O <: SemObserved, - I <: SemImplied, - L <: SemLoss, - D <: SemOptimizer} <: AbstractSemSingle{O, I, L, D} +struct SemFiniteDiff{O <: SemObserved, I <: SemImplied, L <: SemLoss} <: + AbstractSemSingle{O, I, L} observed::O implied::I loss::L - optimizer::D end ``` -Additionally, we need to define a method to compute at least the objective value, and if you want to use gradient based optimizers (which you most probably will), we need also to define a method to compute the gradient. For example, the respective fallback methods for all `AbstractSemSingle` models are defined as +Additionally, you can change how objective/gradient/hessian values are computed by providing methods for `evaluate!`, e.g. from `SemFiniteDiff`'s implementation: ```julia -function objective!(model::AbstractSemSingle, parameters) - objective!(implied(model), parameters, model) - return objective!(loss(model), parameters, model) -end - -function gradient!(gradient, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - gradient!(implied(model), parameters, model) - gradient!(gradient, loss(model), parameters, model) -end +evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) = ... ``` -Note that the `gradient!` method takes a pre-allocated array that should be filled with the gradient values. - Additionally, we can define constructors like the one in `"src/frontend/specification/Sem.jl"`. It is also possible to add new subtypes for `AbstractSemCollection`. \ No newline at end of file diff --git a/docs/src/internals/files.md b/docs/src/internals/files.md index 9cf455fdc..0872c2b02 100644 --- a/docs/src/internals/files.md +++ b/docs/src/internals/files.md @@ -4,7 +4,7 @@ We briefly describe the file and folder structure of the package. ## Source code -All source code is in the `"src"` folder: +Source code is in the `"src"` folder: `"src"` - `"StructuralEquationModels.jl"` defines the module and the exported objects @@ -13,12 +13,15 @@ All source code is in the `"src"` folder: - The four folders `"observed"`, `"implied"`, `"loss"` and `"diff"` contain implementations of specific subtypes (for example, the `"loss"` folder contains a file `"ML.jl"` that implements the `SemML` loss function). - `"optimizer"` contains connections to different optimization backends (aka methods for `sem_fit`) - `"optim.jl"`: connection to the `Optim.jl` package - - `"NLopt.jl"`: connection to the `NLopt.jl` package - `"frontend"` contains user-facing functions - `"specification"` contains functionality for model specification - `"fit"` contains functionality for model assessment, like fit measures and standard errors - `"additional_functions"` contains helper functions for simulations, loading artifacts (example data) and various other things +Code for the package extentions can be found in the `"ext"` folder: +- `"SEMNLOptExt"` for connection to `NLopt.jl`. +- `"SEMProximalOptExt"` for connection to `ProximalAlgorithms.jl`. + ## Tests and Documentation Tests are in the `"test"` folder, documentation in the `"docs"` folder. \ No newline at end of file diff --git a/docs/src/internals/types.md b/docs/src/internals/types.md index 980d0f42f..e70a52ca4 100644 --- a/docs/src/internals/types.md +++ b/docs/src/internals/types.md @@ -3,11 +3,11 @@ The type hierarchy is implemented in `"src/types.jl"`. `AbstractSem`: the most abstract type in our package -- `AbstractSemSingle{O, I, L, D} <: AbstractSem` is an abstract parametric type that is a supertype of all single models +- `AbstractSemSingle{O, I, L} <: AbstractSem` is an abstract parametric type that is a supertype of all single models - `Sem`: models that do not need automatic differentiation or finite difference approximation - `SemFiniteDiff`: models whose gradients and/or hessians should be computed via finite difference approximation - `AbstractSemCollection <: AbstractSem` is an abstract supertype of all models that contain multiple `AbstractSem` submodels -Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, `SemLoss` and `SemOptimizer` fields (and can have additional fields). +Every `AbstractSemSingle` has to have `SemObserved`, `SemImplied`, and `SemLoss` fields (and can have additional fields). `SemLoss` is a container for multiple `SemLossFunctions`. \ No newline at end of file diff --git a/docs/src/performance/simulation.md b/docs/src/performance/simulation.md index e46be64a7..881da6222 100644 --- a/docs/src/performance/simulation.md +++ b/docs/src/performance/simulation.md @@ -4,7 +4,7 @@ We are currently working on an interface for simulation studies. Until we are finished with this, this page is just a collection of tips. -## Swap observed data +## Replace observed data In simulation studies, a common task is fitting the same model to many different datasets. It would be a waste of resources to reconstruct the complete model for each dataset. We therefore provide the function `replace_observed` to change the `observed` part of a model, @@ -64,25 +64,44 @@ model = Sem( model_updated = replace_observed(model; data = data_2, specification = partable) ``` +If you are building your models by parts, you can also update each part seperately with the function `update_observed`. +For example, + +```@example replace_observed + +new_observed = SemObservedData(;data = data_2, specification = partable) + +my_optimizer = SemOptimizerOptim() + +new_optimizer = update_observed(my_optimizer, new_observed) +``` + +## Multithreading !!! danger "Thread safety" *This is only relevant when you are planning to fit updated models in parallel* - Models generated this way may share the same objects in memory (e.g. some parts of + Models generated by `replace_observed` may share the same objects in memory (e.g. some parts of `model` and `model_updated` are the same objects in memory.) Therefore, fitting both of these models in parallel will lead to **race conditions**, possibly crashing your computer. To avoid these problems, you should copy `model` before updating it. -If you are building your models by parts, you can also update each part seperately with the function `update_observed`. -For example, +Taking into account the warning above, fitting multiple models in parallel becomes as easy as: -```@example replace_observed +```julia +model1 = Sem( + specification = partable, + data = data_1 +) -new_observed = SemObservedData(;data = data_2, specification = partable) +model2 = deepcopy(replace_observed(model; data = data_2, specification = partable)) -my_optimizer = SemOptimizerOptim() +models = [model1, model2] +fits = Vector{SemFit}(undef, 2) -new_optimizer = update_observed(my_optimizer, new_observed) +Threads.@threads for i in 1:2 + fits[i] = sem_fit(models[i]) +end ``` ## API diff --git a/docs/src/tutorials/backends/nlopt.md b/docs/src/tutorials/backends/nlopt.md index f861e174e..2afa5e547 100644 --- a/docs/src/tutorials/backends/nlopt.md +++ b/docs/src/tutorials/backends/nlopt.md @@ -35,6 +35,8 @@ my_optimizer = SemOptimizerNLopt(; This uses an augmented lagrangian method with LBFGS as the local optimization algorithm, stops at a maximum of 200 evaluations and uses a relative tolerance of the objective value of `1e-6` as the stopping criterion for the local algorithm. +To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. + In the NLopt docs, you can find explanations about the different [algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/) and a [tutorial](https://nlopt.readthedocs.io/en/latest/NLopt_Introduction/) that also explains the different options. To choose an algorithm, just pass its name without the 'NLOPT\_' prefix (for example, 'NLOPT\_LD\_SLSQP' can be used by passing `algorithm = :LD_SLSQP`). diff --git a/docs/src/tutorials/backends/optim.md b/docs/src/tutorials/backends/optim.md index aaaf4ac9b..cf287e773 100644 --- a/docs/src/tutorials/backends/optim.md +++ b/docs/src/tutorials/backends/optim.md @@ -17,6 +17,8 @@ my_optimizer = SemOptimizerOptim( ) ``` -A model with this optimizer object will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console. +This optimizer will use BFGS (!not L-BFGS) with a back tracking linesearch and a certain initial step length guess. Also, the trace of the optimization will be printed to the console. + +To see how to use the optimizer to actually fit a model now, check out the [Model fitting](@ref) section. For a list of all available algorithms and options, we refer to [this page](https://julianlsolvers.github.io/Optim.jl/stable/#user/config/) of the `Optim.jl` manual. \ No newline at end of file diff --git a/docs/src/tutorials/collection/collection.md b/docs/src/tutorials/collection/collection.md index 84fa00500..f60b7312c 100644 --- a/docs/src/tutorials/collection/collection.md +++ b/docs/src/tutorials/collection/collection.md @@ -15,11 +15,10 @@ model_2 = SemFiniteDiff(...) model_3 = Sem(...) -model_ensemble = SemEnsemble(model_1, model_2, model_3; optimizer = ...) +model_ensemble = SemEnsemble(model_1, model_2, model_3) ``` So you just construct the individual models (however you like) and pass them to `SemEnsemble`. -One important thing to note is that the individual optimizer entries of each model do not matter (as you can optimize your ensemble model only with one algorithmn from one optimization suite). Instead, `SemEnsemble` has its own optimizer part that specifies the backend for the whole ensemble model. You may also pass a vector of weigths to `SemEnsemble`. By default, those are set to ``N_{model}/N_{total}``, i.e. each model is weighted by the number of observations in it's data (which matches the formula for multigroup models). Multigroup models can also be specified via the graph interface; for an example, see [Multigroup models](@ref). diff --git a/docs/src/tutorials/collection/multigroup.md b/docs/src/tutorials/collection/multigroup.md index d0fc71796..23c13b950 100644 --- a/docs/src/tutorials/collection/multigroup.md +++ b/docs/src/tutorials/collection/multigroup.md @@ -81,8 +81,8 @@ model_ml_multigroup = SemEnsemble( We now fit the model and inspect the parameter estimates: ```@example mg; ansicolor = true -solution = sem_fit(model_ml_multigroup) -update_estimate!(partable, solution) +fit = sem_fit(model_ml_multigroup) +update_estimate!(partable, fit) details(partable) ``` diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index ffd83d4e0..b1fff82b8 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -148,11 +148,10 @@ In this example, we set both tolerances to `1e-8`. ```@example constraints model_constrained = Sem( specification = partable, - data = data, - optimizer = constrained_optimizer + data = data ) -model_fit_constrained = sem_fit(model_constrained) +model_fit_constrained = sem_fit(constrained_optimizer, model_constrained) ``` As you can see, the optimizer converged (`:XTOL_REACHED`) and investigating the solution yields diff --git a/docs/src/tutorials/construction/outer_constructor.md b/docs/src/tutorials/construction/outer_constructor.md index 7de3d9e61..6a3cd2cef 100644 --- a/docs/src/tutorials/construction/outer_constructor.md +++ b/docs/src/tutorials/construction/outer_constructor.md @@ -21,7 +21,7 @@ Structural Equation Model The output of this call tells you exactly what model you just constructed (i.e. what the loss functions, observed, implied and optimizer parts are). -As you can see, by default, we use maximum likelihood estimation, the RAM implied type and the `Optim.jl` optimization backend. +As you can see, by default, we use maximum likelihood estimation abd the RAM implied type. To choose something different, you can provide it as a keyword argument: ```julia @@ -31,11 +31,10 @@ model = Sem( observed = ..., implied = ..., loss = ..., - optimizer = ... ) ``` -For example, to construct a model for weighted least squares estimation that uses symbolic precomputation and the Optim backend, write +For example, to construct a model for weighted least squares estimation that uses symbolic precomputation, write ```julia model = Sem( diff --git a/docs/src/tutorials/fitting/fitting.md b/docs/src/tutorials/fitting/fitting.md index b534ad754..a3e4b9b91 100644 --- a/docs/src/tutorials/fitting/fitting.md +++ b/docs/src/tutorials/fitting/fitting.md @@ -43,7 +43,29 @@ Structural Equation Model ∇f(x) calls: 524 ``` -You may optionally specify [Starting values](@ref). +## Choosing an optimizer + +To choose a different optimizer, you can call `sem_fit` with the keyword argument `engine = ...`, and pass additional keyword arguments: + +```julia +using Optim + +model_fit = sem_fit(model; engine = :Optim, algorithm = BFGS()) +``` + +Available options for engine are `:Optim`, `:NLopt` and `:Proximal`, where `:NLopt` and `:Proximal` are only available if the `NLopt.jl` and `ProximalAlgorithms.jl` packages are loaded respectively. + +The available keyword arguments are listed in the sections [Using Optim.jl](@ref), [Using NLopt.jl](@ref) and [Regularization](@ref). + +Alternative, you can also explicitely define a `SemOptimizer` and pass it as the first argument to `sem_fit`: + +```julia +my_optimizer = SemOptimizerOptim(algorithm = BFGS()) + +sem_fit(my_optimizer, model) +``` + +You may also optionally specify [Starting values](@ref). # API - model fitting diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index 692f6cebc..dd5a7f171 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -35,7 +35,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 # means - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable = ParameterTable( @@ -73,7 +73,7 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 # means - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable = ParameterTable( @@ -99,7 +99,7 @@ model = Sem( sem_fit(model) ``` -If we build the model by parts, we have to pass the `meanstructure = true` argument to every part that requires it (when in doubt, simply comsult the documentation for the respective part). +If we build the model by parts, we have to pass the `meanstructure = true` argument to every part that requires it (when in doubt, simply consult the documentation for the respective part). For our example, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 89fb6d151..27d58f93f 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -11,7 +11,7 @@ Return a new model with swaped observed part. - `observed`: Either an object of subtype of `SemObserved` or a subtype of `SemObserved` # Examples -See the online documentation on [Swap observed data](@ref). +See the online documentation on [Replace observed data](@ref). """ function replace_observed end @@ -21,7 +21,7 @@ function replace_observed end Update a `SemImplied`, `SemLossFunction` or `SemOptimizer` object to use a `SemObserved` object. # Examples -See the online documentation on [Swap observed data](@ref). +See the online documentation on [Replace observed data](@ref). # Implementation You can provide a method for this function when defining a new type, for more information diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 70bf6816c..8ee134a9c 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -212,7 +212,7 @@ function details( ) print("\n") - mean_indices = findall(r -> (r.relation == :→) && (r.from == Symbol("1")), partable) + mean_indices = findall(r -> (r.relation == :→) && (r.from == Symbol(1)), partable) if length(mean_indices) > 0 printstyled("Means: \n"; color = color) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index c5ad010b3..74c963ccb 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -197,7 +197,7 @@ function sort_vars!(partable::ParameterTable) partable.columns[:relation], partable.columns[:from], partable.columns[:to], - ) if (rel == :→) && (from != Symbol("1")) + ) if (rel == :→) && (from != Symbol(1)) ] sort!(edges, by = last) # sort edges by target @@ -492,7 +492,7 @@ function lavaan_param_values!( ) lav_ind = nothing - if from == Symbol("1") + if from == Symbol(1) lav_ind = findallrows( r -> r[:lhs] == String(to) && diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 43fd87945..4ebea95fb 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -154,16 +154,16 @@ function RAMMatrices( S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? M_inds = - any(==(Symbol("1")), partable.columns[:from]) ? + any(==(Symbol(1)), partable.columns[:from]) ? [Vector{Int64}() for _ in 1:length(params)] : nothing M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for r in partable row_ind = vars_index[r.to] - col_ind = r.from != Symbol("1") ? vars_index[r.from] : nothing + col_ind = r.from != Symbol(1) ? vars_index[r.from] : nothing if !r.free - if (r.relation == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol(1)) push!(M_consts, row_ind => r.value_fixed) elseif r.relation == :→ push!( @@ -186,7 +186,7 @@ function RAMMatrices( end else par_ind = params_index[r.param] - if (r.relation == :→) && (r.from == Symbol("1")) + if (r.relation == :→) && (r.from == Symbol(1)) push!(M_inds[par_ind], row_ind) elseif r.relation == :→ push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) @@ -328,7 +328,7 @@ function partable_row( # variable names if matrix == :M - from = Symbol("1") + from = Symbol(1) to = varnames[index] else from = varnames[index[2]] diff --git a/src/implied/empty.jl b/src/implied/empty.jl index e87dc72d1..11cc579a4 100644 --- a/src/implied/empty.jl +++ b/src/implied/empty.jl @@ -25,17 +25,17 @@ model per group and an additional model with `ImpliedEmpty` and `SemRidge` for t ## Implementation Subtype of `SemImplied`. """ -struct ImpliedEmpty{V2} <: SemImplied - hessianeval::ExactHessian - meanstruct::NoMeanStruct - ram_matrices::V2 +struct ImpliedEmpty{A, B, C} <: SemImplied + hessianeval::A + meanstruct::B + ram_matrices::C end ############################################################################################ ### Constructors ############################################################################################ -function ImpliedEmpty(; specification, kwargs...) +function ImpliedEmpty(;specification, meanstruct = NoMeanStruct(), hessianeval = ExactHessian(), kwargs...) return ImpliedEmpty(hessianeval, meanstruct, convert(RAMMatrices, specification)) end diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 2f5135176..1e97617fc 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -11,7 +11,7 @@ model_g2 = Sem(specification = specification_g2, data = dat_g2, implied = RAM) @test SEM.params(model_g1.implied.ram_matrices) == SEM.params(model_g2.implied.ram_matrices) # test the different constructors -model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) +model_ml_multigroup = SemEnsemble(model_g1, model_g2) model_ml_multigroup2 = SemEnsemble( specification = partable, data = dat, @@ -28,7 +28,7 @@ end # fit @testset "ml_solution_multigroup" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) update_estimate!(partable, solution) test_estimates( partable, @@ -36,7 +36,7 @@ end atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable, solution) test_estimates( partable, @@ -268,7 +268,6 @@ if !isnothing(specification_miss_g1) loss = SemFIML, data = dat_miss_g1, implied = RAM, - optimizer = SemOptimizerEmpty(), meanstructure = true, ) @@ -278,11 +277,10 @@ if !isnothing(specification_miss_g1) loss = SemFIML, data = dat_miss_g2, implied = RAM, - optimizer = SemOptimizerEmpty(), meanstructure = true, ) - model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) + model_ml_multigroup = SemEnsemble(model_g1, model_g2) model_ml_multigroup2 = SemEnsemble( specification = partable_miss, data = dat_missing, @@ -323,7 +321,7 @@ if !isnothing(specification_miss_g1) end @testset "fiml_solution_multigroup" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -331,7 +329,7 @@ if !isnothing(specification_miss_g1) atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) update_estimate!(partable_miss, solution) test_estimates( partable_miss, @@ -342,7 +340,7 @@ if !isnothing(specification_miss_g1) end @testset "fitmeasures/se_fiml" begin - solution = sem_fit(model_ml_multigroup) + solution = sem_fit(semoptimizer, model_ml_multigroup) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; @@ -359,7 +357,7 @@ if !isnothing(specification_miss_g1) lav_groups = Dict(:Pasteur => 1, :Grant_White => 2), ) - solution = sem_fit(model_ml_multigroup2) + solution = sem_fit(semoptimizer, model_ml_multigroup2) test_fitmeasures( fit_measures(solution), solution_lav[:fitmeasures_fiml]; diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 7dd871ac2..eac2b38dd 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -86,7 +86,7 @@ start_test = [ fill(0.05, 3) fill(0.01, 3) ] -semoptimizer = SemOptimizerOptim +semoptimizer = SemOptimizerOptim() @testset "RAMMatrices | constructor | Optim" begin include("build_models.jl") @@ -137,7 +137,7 @@ graph = @StenoGraph begin _(observed_vars) ↔ _(observed_vars) _(latent_vars) ⇔ _(latent_vars) - Symbol("1") → _(observed_vars) + Symbol(1) → _(observed_vars) end partable_miss = EnsembleParameterTable( @@ -169,7 +169,7 @@ start_test = [ 0.01 0.05 ] -semoptimizer = SemOptimizerOptim +semoptimizer = SemOptimizerOptim() @testset "Graph → Partable → RAMMatrices | constructor | Optim" begin include("build_models.jl") diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 9d026fb28..7394175b7 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -216,8 +216,8 @@ graph = @StenoGraph begin y3 ↔ y7 y8 ↔ y4 + y6 # means - Symbol("1") → _(mean_labels) .* _(observed_vars) - Symbol("1") → fixed(0) * ind60 + Symbol(1) → _(mean_labels) .* _(observed_vars) + Symbol(1) → fixed(0) * ind60 end spec_mean = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) From b91f25a0e88355b5880118ad8400667a5588a613 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 13:50:56 +0100 Subject: [PATCH 65/71] rm ProximalSEM from docs deps --- docs/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 2daded98f..42f6718a9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -4,4 +4,3 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" -ProximalSEM = "3652f839-8142-48b2-a17c-985bd14407c5" From 56650e792cdef3056aae13e45fab038ec9f8517b Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 14:09:13 +0100 Subject: [PATCH 66/71] fix docs --- docs/src/tutorials/concept.md | 2 ++ docs/src/tutorials/constraints/constraints.md | 2 ++ docs/src/tutorials/construction/build_by_parts.md | 2 +- docs/src/tutorials/inspection/inspection.md | 1 + docs/src/tutorials/meanstructure.md | 2 +- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index d663d3c2c..e4d116877 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -72,6 +72,8 @@ SemObserved SemObservedData SemObservedCovariance SemObservedMissing +samples +SemSpecification ``` ## implied diff --git a/docs/src/tutorials/constraints/constraints.md b/docs/src/tutorials/constraints/constraints.md index b1fff82b8..cdd9111a2 100644 --- a/docs/src/tutorials/constraints/constraints.md +++ b/docs/src/tutorials/constraints/constraints.md @@ -122,6 +122,8 @@ In NLopt, vector-valued constraints are also possible, but we refer to the docum We now have everything together to specify and fit our model. First, we specify our optimizer backend as ```@example constraints +using NLopt + constrained_optimizer = SemOptimizerNLopt( algorithm = :AUGLAG, options = Dict(:upper_bounds => upper_bounds, :xtol_abs => 1e-4), diff --git a/docs/src/tutorials/construction/build_by_parts.md b/docs/src/tutorials/construction/build_by_parts.md index 27604d2a1..606a6576e 100644 --- a/docs/src/tutorials/construction/build_by_parts.md +++ b/docs/src/tutorials/construction/build_by_parts.md @@ -59,7 +59,7 @@ ml = SemML(observed = observed) loss_ml = SemLoss(ml) # optimizer ------------------------------------------------------------------------------------- -optimizer = SemOptimizerOptim(algorithm = BFGS()) +optimizer = SemOptimizerOptim() # model ------------------------------------------------------------------------------------ diff --git a/docs/src/tutorials/inspection/inspection.md b/docs/src/tutorials/inspection/inspection.md index faab8f8ed..2b6d3191f 100644 --- a/docs/src/tutorials/inspection/inspection.md +++ b/docs/src/tutorials/inspection/inspection.md @@ -130,6 +130,7 @@ df minus2ll nobserved_vars nsamples +params nparams p_value RMSEA diff --git a/docs/src/tutorials/meanstructure.md b/docs/src/tutorials/meanstructure.md index dd5a7f171..60578224a 100644 --- a/docs/src/tutorials/meanstructure.md +++ b/docs/src/tutorials/meanstructure.md @@ -110,7 +110,7 @@ implied_ram = RAM(specification = partable, meanstructure = true) ml = SemML(observed = observed, meanstructure = true) -model = Sem(observed, implied_ram, SemLoss(ml), SemOptimizerOptim()) +model = Sem(observed, implied_ram, SemLoss(ml)) sem_fit(model) ``` \ No newline at end of file From 0f09635652770f882434ccffab18fe026e8959d7 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 14:17:44 +0100 Subject: [PATCH 67/71] fix docs --- docs/src/tutorials/concept.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/tutorials/concept.md b/docs/src/tutorials/concept.md index e4d116877..035144d62 100644 --- a/docs/src/tutorials/concept.md +++ b/docs/src/tutorials/concept.md @@ -73,6 +73,7 @@ SemObservedData SemObservedCovariance SemObservedMissing samples +observed_vars SemSpecification ``` From 5cc61bb0eca593746a51450cd0c8f638545f64d0 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:11:32 +0100 Subject: [PATCH 68/71] try to fix svgs for docs --- docs/src/assets/concept.svg | 17 +++++++++++++++++ docs/src/assets/concept_typed.svg | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index 138463b67..c5a3c6bb6 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -6,6 +6,7 @@ stroke="none" stroke-linecap="square" stroke-miterlimit="10" +<<<<<<< Updated upstream id="svg23" sodipodi:docname="Unbenannte Präsentation (2).svg" width="921600" @@ -34,6 +35,22 @@ margin="0" bleed="0" /> +======= + id="svg57" + width="610.56537" + height="300.26614" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + + + + +>>>>>>> Stashed changes +======= + id="svg57" + width="610.56537" + height="300.26614" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + + + + +>>>>>>> Stashed changes Date: Tue, 4 Feb 2025 15:13:04 +0100 Subject: [PATCH 69/71] try to fix svgs for docs --- docs/src/assets/concept.svg | 136 +++++++++++++---------------- docs/src/assets/concept_typed.svg | 139 +++++++++++++----------------- 2 files changed, 124 insertions(+), 151 deletions(-) diff --git a/docs/src/assets/concept.svg b/docs/src/assets/concept.svg index c5a3c6bb6..fa222a0d9 100644 --- a/docs/src/assets/concept.svg +++ b/docs/src/assets/concept.svg @@ -1,41 +1,11 @@ - - - - -======= id="svg57" width="610.56537" height="300.26614" @@ -50,134 +20,152 @@ clip-rule="nonzero" id="path2" /> ->>>>>>> Stashed changes + id="path7" /> + id="path9" /> + id="path11" /> + id="path13" /> + id="path15" /> + id="path17" /> + id="path19" /> + id="path21" /> + id="path23" /> + id="path25" /> + id="path27" /> + id="path29" /> + id="path31" /> + id="path33" /> + id="path35" /> + id="path37" /> + id="path39" /> + id="path41" /> + id="path43" /> + id="path45" /> + id="path47" /> + + + diff --git a/docs/src/assets/concept_typed.svg b/docs/src/assets/concept_typed.svg index 9b2d72305..88a0d8566 100644 --- a/docs/src/assets/concept_typed.svg +++ b/docs/src/assets/concept_typed.svg @@ -1,44 +1,11 @@ - - - - -======= id="svg57" width="610.56537" height="300.26614" @@ -53,134 +20,152 @@ clip-rule="nonzero" id="path2" /> ->>>>>>> Stashed changes + id="path7" /> + id="path9" /> + id="path11" /> + id="path13" /> + id="path15" /> + id="path17" /> + id="path19" /> + id="path21" /> + id="path23" /> + id="path25" /> + id="path27" /> + id="path29" /> + id="path31" /> + id="path33" /> + id="path35" /> + id="path37" /> + id="path39" /> + id="path41" /> + id="path43" /> + id="path45" /> + id="path47" /> + + + From b32701263b91959c1882dc07df40ab3dab7f64ce Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:33:26 +0100 Subject: [PATCH 70/71] update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3eeafd332..79c11da21 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ It is still *in development*. Models you can fit include - Linear SEM that can be specified in RAM (or LISREL) notation - ML, GLS and FIML estimation -- Regularization +- Regularized SEM (Ridge, Lasso, L0, ...) - Multigroup SEM - Sums of arbitrary loss functions (everything the optimizer can handle). @@ -35,6 +35,7 @@ The package makes use of - Symbolics.jl for symbolically precomputing parts of the objective and gradients to generate fast, specialized functions. - SparseArrays.jl to speed up symbolic computations. - Optim.jl and NLopt.jl to provide a range of different Optimizers/Linesearches. +- ProximalAlgorithms.jl for regularization. - FiniteDiff.jl and ForwardDiff.jl to provide gradients for user-defined loss functions. # At the moment, we are still working on: From 4091804cca8c43d6eac6784fe489b61649ace1b3 Mon Sep 17 00:00:00 2001 From: Maximilian Ernst Date: Tue, 4 Feb 2025 15:38:04 +0100 Subject: [PATCH 71/71] bump version --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index d55346aca..94ab214e8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StructuralEquationModels" uuid = "383ca8c5-e4ff-4104-b0a9-f7b279deed53" authors = ["Maximilian Ernst", "Aaron Peikert"] -version = "0.2.4" +version = "0.3.0" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" @@ -24,7 +24,7 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" [compat] -julia = "1.9, 1.10" +julia = "1.9, 1.10, 1.11" StenoGraphs = "0.2 - 0.3, 0.4.1 - 0.5" DataFrames = "1" Distributions = "0.25"