lazy compilation for grad_cv_operator

[anderson2981]

forced compilation of grad_cv_operator outside of the rhs appears to cause compilation errors.

See for example test.py in the physical_av branch. We add a function

    from mirgecom.navierstokes import grad_cv_operator
    quadrature_tag = None

    # compiled wrapper for grad_cv_operator
    def _grad_cv_operator(fluid_state, time):
        return grad_cv_operator(discr=discr, gas_model=gas_model,
                                boundaries=boundaries,
                                state=fluid_state,
                                time=time,
                                quadrature_tag=quadrature_tag)

    grad_cv_operator_compiled = actx.compile(_grad_cv_operator)

and call it from my_pre_step,

Running with

mpirun -n 1 python -u -O -m mpi4py test.py --lazy

generates an error:

28 errors generated.
Traceback (most recent call last):
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/runpy.py", line 197, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/mpi4py/__main__.py", line 7, in <module>
    main()
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/mpi4py/run.py", line 198, in main
    run_command_line(args)
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/mpi4py/run.py", line 47, in run_command_line
    run_path(sys.argv[0], run_name='__main__')
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/runpy.py", line 288, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/runpy.py", line 97, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "test.py", line 416, in <module>
    main(actx_class, use_logmgr=args.log, use_leap=args.leap, lazy=lazy,
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/mirgecom/mirgecom/mpi.py", line 157, in wrapped_func
    func(*args, **kwargs)
  File "test.py", line 358, in main
    advance_state(rhs=my_rhs, timestepper=timestepper,
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/mirgecom/mirgecom/steppers.py", line 419, in advance_state
    _advance_state_stepper_func(
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/mirgecom/mirgecom/steppers.py", line 154, in _advance_state_stepper_func
    state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt)
  File "test.py", line 297, in my_pre_step
    grad_cv = grad_cv_operator_compiled(fluid_state, time=t)
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/arraycontext/arraycontext/impl/pytato/compile.py", line 367, in __call__
    return compiled_func(arg_id_to_arg)
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/grudge/grudge/array_context.py", line 363, in __call__
    out_dict = execute_distributed_partition(
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/pytato/pytato/distributed.py", line 1222, in execute_distributed_partition
    exec_ready_part(part)
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/pytato/pytato/distributed.py", line 1176, in exec_ready_part
    _evt, result_dict = prg_per_partition[part.pid](queue,
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/pytato/pytato/target/loopy/__init__.py", line 158, in __call__
    return self.program(queue,
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/loopy/loopy/translation_unit.py", line 347, in __call__
    return pex(*args, **kwargs)
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/loopy/loopy/target/pyopencl_execution.py", line 384, in __call__
    translation_unit_info = self.translation_unit_info(entrypoint,
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/pytools/__init__.py", line 766, in wrapper
    result = function(obj, *args, **kwargs)
  File "/Users/someguy/work/CEESD/MirgeCom/Drivers/CEESD-Y2_injector/emirge/loopy/loopy/target/pyopencl_execution.py", line 332, in translation_unit_info
    cl.Program(self.context, dev_code)
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/pyopencl/__init__.py", line 539, in build
    self._prg, was_cached = self._build_and_catch_errors(
  File "/Users/someguy/software/Install/Conda/envs/mirgeDriver.Y2injector/lib/python3.9/site-packages/pyopencl/__init__.py", line 587, in _build_and_catch_errors
    raise err
pyopencl._cl.RuntimeError: clBuildProgram failed: BUILD_PROGRAM_FAILURE - clBuildProgram failed: BUILD_PROGRAM_FAILURE - clBuildProgram failed: BUILD_PROGRAM_FAILURE

Build on <pyopencl.Device 'pthread' on 'Portable Computing Language' at 0x148432400>:

error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:186:133: expected parameter declarator
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:186:133: expected ')'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:186:138: expected ')'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:186:115: pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:188:3: use of undeclared identifier '_pt_out_energy_0_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:188:69: use of undeclared identifier '_pt_temp_14'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:189:3: use of undeclared identifier '_pt_out_energy_1_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:189:69: use of undeclared identifier '_pt_temp_12'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:190:3: use of undeclared identifier '_pt_out_mass_0_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:190:67: use of undeclared identifier '_pt_temp_10'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:191:3: use of undeclared identifier '_pt_out_mass_1_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:191:67: use of undeclared identifier '_pt_temp_8'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:192:21: too many arguments to function call, expected 0, have 2
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:192:26: expected ';' after expression
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:192:26: use of undeclared identifier '_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:192:76: use of undeclared identifier '_pt_temp_6'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:193:21: too many arguments to function call, expected 0, have 2
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:193:26: expected ';' after expression
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:193:26: use of undeclared identifier '_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:193:76: use of undeclared identifier '_pt_temp_4'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:194:21: too many arguments to function call, expected 0, have 2
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:194:26: expected ';' after expression
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:194:26: use of undeclared identifier '_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:194:76: use of undeclared identifier '_pt_temp_2'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:195:21: too many arguments to function call, expected 0, have 2
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:195:26: expected ';' after expression
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:195:26: use of undeclared identifier '_0'
error: /Users/someguy/.cache/pocl/kcache/tempfile_hRrJEx.cl:195:76: use of undeclared identifier '_pt_temp_0'
Device pthread failed to build the program