Defining a solver type with macros

juliohm · September 10, 2017, 10:07pm

I am trying to write a macro that defines a solver type with a set of parameters. The idea is to write something like the following:

@simsolver MySolver begin
  @param a = 1.0
  @param b = false
  @param c
end

and get:

@with_kw struct MySolverParam
  a = 1.0
  b =  false
  c
end

struct MySolver
  params::Dict{Symbol,MySolverParam}

  MySolver(params::Dict{Symbol,MySolverParam}) = new(params)
end

function MySolver(params...)
  # build dictionary for inner constructor
  dict = Dict{Symbol,MySolverParam}()

  # convert named tuples to solver parameters
  for (varname, varparams) in params
    kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
    push!(dict, varname => MySolverParam(; kwargs...))
  end

  MySolver(dict)
end

where @with_kw is defined in Parameters.jl.

I got stuck on my initial attempt:

using Parameters

macro simsolver(solver, body)
    paramtype = esc(Symbol(solver,"Param"))
    quote
        struct $paramtype
            # how to copy every line starting with @param to here?
        end
    
        struct $solver
            params::Dict{Symbol,$paramtype}
    
            $solver(params::Dict{Symbol,$paramtype}) = new(params)
        end

        # this outer constructor is not visible for some reason?
        function $solver(params...)
            # build dictionary for inner constructor
            dict = Dict{Symbol,$paramtype}()

            # convert named tuples to solver parameters
            for (varname, varparams) in params
                kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                push!(dict, varname => $paramtype(; kwargs...))
            end

            $solver(dict)
        end
    end
end

The outer constructor is not visible for some reason and I don’t know how I could copy every line starting with @param to the body of struct @paramtype end. I appreciate any help…

favba · September 11, 2017, 4:32am

I recommend you read the hygiene section in the docs.
I also recommend (again) using @macroexpand to see what your macro is doing so you can look for what to change. The output of the macro should be the a quoted code that you originally wanted.
If I use your macro as defined and test on the example you provided this is what we get:

julia> @macroexpand @simsolver MySolver begin
         @param a = 1.0
         @param b = false
         @param c
       end
quote  # REPL[15], line 4:
    struct MySolverParam # REPL[15], line 6:
    end # REPL[15], line 8:
    struct MySolver # REPL[15], line 9:
        params::(Main.Dict){Main.Symbol, MySolverParam} # REPL[15], line 11:
        MySolver(#84#params::(Main.Dict){Main.Symbol, MySolverParam}) = begin  # REPL[15], line 11:
                new(#84#params)
            end
    end # REPL[15], line 15:
    function #82#MySolver(#91#params...) # REPL[15], line 17:
        #85#dict = (Main.Dict){Main.Symbol, MySolverParam}() # REPL[15], line 20:
        for (#86#varname, #87#varparams) = #91#params # REPL[15], line 21:
            #88#kwargs = [#89#k => #90#v for (#89#k, #90#v) = (Main.zip)((Main.keys)(#87#varparams), #87#varparams)] # REPL[15], line 22:
            (Main.push!)(#85#dict, #86#varname => MySolverParam(; #88#kwargs...))
        end # REPL[15], line 25:
        #82#MySolver(#85#dict)
    end
end

All those #SomeNumber#Then_what_I_originally_wrote is due to macro hygiene.

Now I’ll change a little bit your macro to:

julia> macro simsolver(solver, body)
           paramtype = Symbol(solver,"Param")
           esc(quote
               struct $paramtype
                   # how to copy every line starting with @param to here?
               end

               struct $solver
                   params::Dict{Symbol,$paramtype}

                   $solver(params::Dict{Symbol,$paramtype}) = new(params)
               end

               # this outer constructor is not visible for some reason?
               function $solver(params...)
                   # build dictionary for inner constructor
                   dict = Dict{Symbol,$paramtype}()

                   # convert named tuples to solver parameters
                   for (varname, varparams) in params
                       kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                       push!(dict, varname => $paramtype(; kwargs...))
                   end

                   $solver(dict)
               end
           end)
       end
@simsolver (macro with 1 method)

Note I’m using esc() on the whole quote. Now testing it again with @macroexpand:

julia> @macroexpand @simsolver MySolver begin
         @param a = 1.0
         @param b = false
         @param c
       end
quote  # REPL[17], line 4:
    struct MySolverParam # REPL[17], line 6:
    end # REPL[17], line 8:
    struct MySolver # REPL[17], line 9:
        params::Dict{Symbol, MySolverParam} # REPL[17], line 11:
        MySolver(params::Dict{Symbol, MySolverParam}) = begin  # REPL[17], line 11:
                new(params)
            end
    end # REPL[17], line 15:
    function MySolver(params...) # REPL[17], line 17:
        dict = Dict{Symbol, MySolverParam}() # REPL[17], line 20:
        for (varname, varparams) = params # REPL[17], line 21:
            kwargs = [k => v for (k, v) = zip(keys(varparams), varparams)] # REPL[17], line 22:
            push!(dict, varname => MySolverParam(; kwargs...))
        end # REPL[17], line 25:
        MySolver(dict)
    end
end

Whoa! Now it is much closer to what we want!

Now, how to copy the list to the struct? I don’t know…
My recommendation is to play a little bit with julia Expr to figure that out. First create a expression for your intended use of the macro:

 julia> ex = :(@simsolver MySolver begin; @param a = 1.0; @param b = false; @param c; end)
:(@simsolver MySolver begin  # REPL[40], line 1:
            @param a = 1.0 # REPL[40], line 1:
            @param b = false # REPL[40], line 1:
            @param c
        end)

Then use the head and args fields of the expression type:

julia> ex.head
:macrocall

julia> ex.args
3-element Array{Any,1}:
 Symbol("@simsolver")
 :MySolver
 quote  # REPL[40], line 1:
    @param a = 1.0 # REPL[40], line 1:
    @param b = false # REPL[40], line 1:
    @param c
end
julia> ex.args[1]
Symbol("@simsolver")

julia> ex.args[2]
:MySolver

julia> ex.args[3]
quote  # REPL[40], line 1:
    @param a = 1.0 # REPL[40], line 1:
    @param b = false # REPL[40], line 1:
    @param c
end

So your expression will generate a macro call (as expected) with three argumetns: The name of the macro and the two parameters provided: the symbol :MySolver and the expression given by the quote.
Now note the third argument is a expression itself. So we can do .head and .args on it also:

julia> ex.args[3].head
:block

julia> ex.args[3].args
6-element Array{Any,1}:
 :( # REPL[40], line 1:)
 :(@param a = 1.0)
 :( # REPL[40], line 1:)
 :(@param b = false)
 :( # REPL[40], line 1:)
 :(@param c)

We are getting there! The arguments are again all Expr themselves. Let’s go to the second one:

julia> ex.args[3].args[2].head
:macrocall

julia> ex.args[3].args[2].args
2-element Array{Any,1}:
 Symbol("@param")
 :(a = 1.0)

There it is, one of the parameters!

julia> ex.args[3].args[2].args[2]
:(a = 1.0)

Now. remember ex.args[3] will be the second argument, named body in the macro definition. So you will access it by doing $(body.args[2].args[2]. Let’s try adding that to your macro definition to see if it works…

julia> macro simsolver(solver, body)
           paramtype = Symbol(solver,"Param")
           esc(quote
               struct $paramtype
                   $(body.args[2].args[2])
               end

               struct $solver
                   params::Dict{Symbol,$paramtype}

                   $solver(params::Dict{Symbol,$paramtype}) = new(params)
               end

               # this outer constructor is not visible for some reason?
               function $solver(params...)
                   # build dictionary for inner constructor
                   dict = Dict{Symbol,$paramtype}()

                   # convert named tuples to solver parameters
                   for (varname, varparams) in params
                       kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                       push!(dict, varname => $paramtype(; kwargs...))
                   end

                   $solver(dict)
               end
           end)
       end
@simsolver (macro with 1 method)

julia> @macroexpand @simsolver MySolver begin
         @param a = 1.0
         @param b = false
         @param c
       end
quote  # REPL[86], line 4:
    struct MySolverParam # REPL[86], line 5:
        a = 1.0
    end # REPL[86], line 8:
    struct MySolver # REPL[86], line 9:
        params::Dict{Symbol, MySolverParam} # REPL[86], line 11:
        MySolver(params::Dict{Symbol, MySolverParam}) = begin  # REPL[86], line 11:
                new(params)
            end
    end # REPL[86], line 15:
    function MySolver(params...) # REPL[86], line 17:
        dict = Dict{Symbol, MySolverParam}() # REPL[86], line 20:
        for (varname, varparams) = params # REPL[86], line 21:
            kwargs = [k => v for (k, v) = zip(keys(varparams), varparams)] # REPL[86], line 22:
            push!(dict, varname => MySolverParam(; kwargs...))
        end # REPL[86], line 25:
        MySolver(dict)
    end
end

There it is! Now, of course, for your actual usage you will have to iterate your body input to look for the Symbol("@param") and do the filtering.

Summarizing:

Use @macroexpand A LOT
Analyze the expressions you are passing to the macro to make your plan to extract the info you want.

Ps:You can also use dump() to analyze expressions but the quantity of info can be overwhelming:

julia> dump(ex)
Expr
  head: Symbol macrocall
  args: Array{Any}((3,))
    1: Symbol @simsolver
    2: Symbol MySolver
    3: Expr
      head: Symbol block
      args: Array{Any}((6,))
        1: Expr
          head: Symbol line
          args: Array{Any}((2,))
            1: Int64 1
            2: Symbol REPL[89]
          typ: Any
        2: Expr
          head: Symbol macrocall
          args: Array{Any}((2,))
            1: Symbol @param
            2: Expr
              head: Symbol =
              args: Array{Any}((2,))
                1: Symbol a
                2: Float64 1.0
              typ: Any
          typ: Any
        3: Expr
          head: Symbol line
          args: Array{Any}((2,))
            1: Int64 1
            2: Symbol REPL[89]
          typ: Any
        4: Expr
          head: Symbol macrocall
          args: Array{Any}((2,))
            1: Symbol @param
            2: Expr
              head: Symbol =
              args: Array{Any}((2,))
                1: Symbol b
                2: Bool false
              typ: Any
          typ: Any
        5: Expr
          head: Symbol line
          args: Array{Any}((2,))
            1: Int64 1
            2: Symbol REPL[89]
          typ: Any
        6: Expr
          head: Symbol macrocall
          args: Array{Any}((2,))
            1: Symbol @param
            2: Symbol c
          typ: Any
      typ: Any
  typ: Any

juliohm · September 11, 2017, 4:41am

That is a beautiful writeup @favba! Thanks for sharing! I will try digest it locally and play around with @macroexpand as you suggested. First time playing with macros, but your answer is helping a lot already to understand the machinery.

juliohm · September 11, 2017, 4:57am

Quick question, these lines with REPL refer to what?

favba · September 11, 2017, 5:05am

I really don’t know. Maybe they’re newline characters? REPL[40] refers to the line (40) of the “file” where the block was written (this case in the julia terminal: Read-Evaluate-Print-Loop).

juliohm · September 11, 2017, 5:07am

That is my guess too In any case, I am really enjoying playing with macros now after your very clear explanation

favba · September 11, 2017, 5:11am

I got all that figured out this past week and was really excited about it. I’m trying to find new places to apply this awesome macro power

favba · September 11, 2017, 5:24am

They definitively refer to the ; used at the expression creation.

juliohm · September 11, 2017, 5:27am

I filtered the parameters prefixed with @param, but now I am wondering how to paste each entry of the vector in the body of the struct, one per line. Do you know a way?

macro simsolver(solver, body)
    solverparam = Symbol(solver,"Param")
    
    body = filter(arg -> arg.head == :macrocall, body.args)
    varparams = filter(p -> p.args[1] == Symbol("@param"), body)
    varparams = map(p -> p.args[2], varparams)
    
    esc(quote
        struct $solverparam
            $varparams # how to split this vector into multiple expressions?
        end
    
        struct $solver
            params::Dict{Symbol,$solverparam}
    
            $solver(params::Dict{Symbol,$solverparam}) = new(params)
        end

        function $solver(params...)
            # build dictionary for inner constructor
            dict = Dict{Symbol,$solverparam}()

            # convert named tuples to solver parameters
            for (varname, varparams) in params
                kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                push!(dict, varname => $solverparam(; kwargs...))
            end

            $solver(dict)
        end
    end)
end

@macroexpand @simsolver MySolver begin
    @param a = 2
    @param c
    @global gpu = false
end

favba · September 11, 2017, 5:43am

I’m not on my pc anymore. But try figuring out how a struct expression is constructed with the same trick above. It will probably be something like ex.head = :struct and then the first argument will be the struct name and the rest the expressions to be evaluated inside the struct. Then you can manually build this particular struct expression and interpolate it inside the returned expression.

kristoffer.carlsson · September 11, 2017, 7:58am

Asa a slightly unrelated note, I’m of the opinion that one should give great thought before making an API based on macros.

From a user’s point of view, as soon as one sees a macro, it is hard to know what will happen since the macro can rewrite the code to whatever it wants. In other words, ones Julia intuition is gone; you are not really programming in Julia anymore, you are programming in a DSL that is made up by the package author.
That can be totally fine (see JuMP) but they have gone to great lengths to provide good documentation and error messages.

Of course, if the “surface” of the macro is small (like @time) then it is usually fine. But unless you are writing a whole DSL, I would avoid macros as much as possible.

juliohm · September 11, 2017, 2:34pm

I fully agree @kristoffer.carlsson, in this case this macro is intended for package contributors only, it is not exported by the package. I want to lower the barrier of getting started with the framework because many people in my field don’t have the necessary background to write advanced constructors for a type, etc. I just want them to paste a simple macro and write a function (i.e. solve).

juliohm · September 12, 2017, 2:20am

I tried joining the entries in varparams as in:

struct $solverparam
    $(join(varparams, '\n'))
end

but it doesn’t do what I need, anyone has an idea on how to split a vector of symbols in this case, one per line? The REPL gives these strange elements :(# In[16], line 4:) which have a head of :line, but I am not sure how I can place a new line inside of the struct.

yuyichao · September 12, 2017, 2:36am

You are NOT generating text so one per line doesn’t make any sense. You are generating expressions and you are splicing in elements or arrays as the expression arguments. Use $(varparams...)

juliohm · September 12, 2017, 2:38am

Thank you @yuyichao! That is awesome, I am almost done with this helper macro. Will work on the latest piece, which is the outer constructor.

juliohm · September 12, 2017, 4:37am

Almost done! Could you please help understand why I am getting this error?

using Parameters

macro simsolver(solver, body)
    solverparam = Symbol(solver,"Param")
    
    # discard lines that doesn't start with @param or @global
    body = filter(arg -> arg.head == :macrocall, body.args)
    
    # lines with @param correspond to variable parameters
    vparams = filter(p -> p.args[1] == Symbol("@param"), body)
    vparams = map(p -> p.args[2], vparams)
    
    # lines with @global are global solver options like verbose, gpu, etc
    gparams = filter(p -> p.args[1] == Symbol("@global"), body)
    gparams = map(p -> p.args[2], gparams)
    
    # these are the global option names without the default values
    gkeys = map(p -> p.args[1], gparams)
    
    esc(quote
        @with_kw_noshow struct $solverparam
            $(vparams...)
        end
    
        struct $solver
            params::Dict{Symbol,$solverparam}
            
            $(gkeys...)
    
            function $solver(params::Dict{Symbol,$solverparam}, $(gkeys...))
                new(params, $(gkeys...))
            end
        end

        function $solver(params...; $(gparams...))
            # build dictionary for inner constructor
            dict = Dict{Symbol,$solverparam}()

            # convert named tuples to solver parameters
            for (varname, varparams) in params
                kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                push!(dict, varname => $solverparam(; kwargs...))
            end

            $solver(dict, $(gkeys...))
        end
    end)
end

@simsolver MySolver begin
    @param a
    @param c
    @global gpu = false
    @global cpu = true
end

ERROR:
syntax: invalid keyword argument syntax "gpu=false" (expected assignment)

When I expand the macro with @macroexpand, everything looks correct in terms of the gpu = false expression, is it because of the double quotes somehow?

kristoffer.carlsson · September 12, 2017, 3:33pm

dump the expression and look at that instead

rdeits · September 12, 2017, 4:59pm

I have a guess as to what’s going on. Keyword arguments are expressions of the form Expr(:kw, :x, 10):

julia> dump(:(f(x=10)))
Expr
  head: Symbol call
  args: Array{Any}((2,))
    1: Symbol f
    2: Expr
      head: Symbol kw
      args: Array{Any}((2,))
        1: Symbol x
        2: Int64 10
      typ: Any
  typ: Any

But your keyword arguments are being parsed as Expr(:=, x, 10):

julia> dump(:(@global gpu = false))
Expr
  head: Symbol macrocall
  args: Array{Any}((2,))
    1: Symbol @global
    2: Expr
      head: Symbol =
      args: Array{Any}((2,))
        1: Symbol gpu
        2: Bool false
      typ: Any
  typ: Any

Does replacing the expression heads with :kw fix your problem?

juliohm · September 12, 2017, 5:15pm

Beautiful @rdeits! Thank you all for the help, I will clean up the code and add it to my dev tools.

juliohm · September 12, 2017, 7:05pm

One last question, how macros play with modules? I tried putting everything together, but as expected the macros paste everything as is and many names are undefined at the call site:

### BASE PACKAGE


module BasePackage
    abstract type AbstractSolver end
    abstract type AbstractEstimationSolver end
    abstract type AbstractSimulationSolver end

    export AbstractEstimationSolver, AbstractSimulationSolver
end # module


#------------------------------------------------------------


### DEVELOPER TOOLS


module DevTools

importall BasePackage
using Parameters: @with_kw_noshow

macro metasolver(solver, solvertype, body)
    solverparam = Symbol(solver,"Param")
    
    # discard any content that doesn't start with @param or @global
    content = filter(arg -> arg.head == :macrocall, body.args)
    
    # lines starting with @param refer to variable parameters
    vparams = filter(p -> p.args[1] == Symbol("@param"), content)
    vparams = map(p -> p.args[2], vparams)
    
    # lines starting with @global refer to global solver parameters
    gparams = filter(p -> p.args[1] == Symbol("@global"), content)
    gparams = map(p -> p.args[2], gparams)
    
    # add default value of `nothing` if necessary
    vparams = map(p -> p isa Symbol ? :($p = nothing) : p, vparams)
    gparams = map(p -> p isa Symbol ? :($p = nothing) : p, gparams)
    
    # replace Expr(:=, a, 2) by Expr(:kw, a, 2) for valid kw args
    gparams = map(p -> Expr(:kw, p.args...), gparams)
    
    # keyword names
    gkeys = map(p -> p.args[1], gparams)
    
    esc(quote
        @with_kw_noshow struct $solverparam
            $(vparams...)
        end
    
        struct $solver <: $solvertype
            params::Dict{Symbol,$solverparam}
            
            $(gkeys...)
    
            function $solver(params::Dict{Symbol,$solverparam}, $(gkeys...))
                new(params, $(gkeys...))
            end
        end

        function $solver(params...; $(gparams...))
            # build dictionary for inner constructor
            dict = Dict{Symbol,$solverparam}()

            # convert named tuples to solver parameters
            for (varname, varparams) in params
                kwargs = [k => v for (k,v) in zip(keys(varparams), varparams)]
                push!(dict, varname => $solverparam(; kwargs...))
            end

            $solver(dict, $(gkeys...))
        end
    end)
end

macro simsolver(solver, body)
    esc(quote
        @metasolver $solver AbstractSimulationSolver $body
    end)
end

macro estimsolver(solver, body)
    esc(quote
        @metasolver $solver AbstractEstimationSolver $body
    end)
end

export @estimsolver, @simsolver

end # module


#------------------------------------------------------------


#### SOLVER DEVELOPER


using DevTools

@simsolver MySolver begin
    @param a = 2
    @param b
    @global gpu = false
end

Names such as @metasolver, @with_kw_noshow are undefined at the call site.