GPU offloading

[benegee]

Very rough summary:

Features

• Support for P4estMesh, except AMR and nonconservative_terms::True
• Methods for GPUs in https://github.com/benegee/Trixi.jl/blob/lc/gpu-develop/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl and https://github.com/benegee/Trixi.jl/blob/lc/gpu-develop/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Important changes

• Adapt.jl, adapt_to for several types
• AbstractHeterogeneousContainer:

  Trixi.jl/src/auxiliary/containers.jl

  Lines 318 to 329 in 33903d9

  	# Containers that support heterogenous computing via KernelAbstractions.jl
  	# should be subtypes of this type.
  	#
  	# The first type parameter must be `Array` by default and if
  	# `Adapt.adapt_structure(to, container)` is called then this must be
  	# `typeof(to)`. This is used in downstream code, e.g. for calling
  	# `wrap_array` with an appropriate type after `resize!`ing.
  	#
  	# The second type parameter determines if KA.jl is used.
  	# By default, each container must initialize this to `false`.
  	# However, when `Adapt.adapt_structure` is called on the container, it must
  	# be changed to `true`.

• VecOfArray

  Trixi.jl/src/auxiliary/vector_of_arrays.jl

  Lines 8 to 13 in 33903d9

  	# Wraps a Vector of Arrays, forwards `getindex` to the underlying Vector.
  	# Implements `Adapt.adapt_structure` to allow offloading to the GPU which is
  	# not possible for a plain Vector of Arrays.
  	struct VecOfArrays{T <: AbstractArray}
  	arrays::Vector{T}
  	end

  used for P4estMPICache

  Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

  Lines 8 to 19 in 33903d9

  	mutable struct P4estMPICache{BufferType <: DenseVector, VecInt <: DenseVector{<:Integer}}
  	mpi_neighbor_ranks::Vector{Int}
  	mpi_neighbor_interfaces::VecOfArrays{VecInt}
  	mpi_neighbor_mortars::VecOfArrays{VecInt}
  	mpi_send_buffers::VecOfArrays{BufferType}
  	mpi_recv_buffers::VecOfArrays{BufferType}
  	mpi_send_requests::Vector{MPI.Request}
  	mpi_recv_requests::Vector{MPI.Request}
  	n_elements_by_rank::OffsetArray{Int, 1, Array{Int, 1}}
  	n_elements_global::Int
  	first_element_global_id::Int
  	end

• Trixi.Indexing module instead of symbols

  Trixi.jl/src/solvers/dg.jl

  Lines 755 to 778 in 33903d9

  	# For some mesh types, elements next to a surface may have local coordinate systems
  	# that are not aligned so the nodes may have to be indexed differently.
  	# `IndexInfo` is used to describe how the nodes should be indexed.
  	# For example, in 2d a `Tuple` with two `IndexInfo` objects, one for each dimension,
  	# would be used.
  	# `first` or `last` indicates that the corresponding index is constant and is either
  	# the first or the last one. This effectively encodes the position of the surface
  	# with respect to the local coordinate system. The other `IndexInfo` object(s)
  	# encode if the index in the corresponding dimension is running forward or backward.
  	#
  	# The Enum is wrapped in a module and exported so that the enum values do not pollute
  	# the global namespace and can only be accessed via `Indexing.value`.
  	module Indexing
  	@enum IndexInfo begin
  	first
  	last
  	i_forward
  	i_backward
  	j_forward
  	j_backward
  	end
  	export IndexInfo
  	end
  	using .Indexing

Todos

• Trixi.jl/src/solvers/dgsem_p4est/containers_parallel.jl

  Line 230 in 33903d9

  # TODO: Vector of Vector type data structure does not work on GPUs,

• Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

  Line 319 in 33903d9

  # TODO reuse existing structures

[github-actions]

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

JuliaFormatter

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_advection_meshview.jl

Line 58 in d942046

x_neg = BoundaryConditionCoupled(2, (Indexing.last, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_advection_meshview.jl

Line 60 in d942046

x_pos = BoundaryConditionCoupled(2, (Indexing.first, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_advection_meshview.jl

Line 66 in d942046

x_neg = BoundaryConditionCoupled(1, (Indexing.last, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_advection_meshview.jl

Line 68 in d942046

x_pos = BoundaryConditionCoupled(1, (Indexing.first, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_mhd_coupled.jl

Line 52 in d942046

boundary_conditions1 = (x_neg = BoundaryConditionCoupled(2, (Indexing.last, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_mhd_coupled.jl

Line 54 in d942046

x_pos = BoundaryConditionCoupled(2, (Indexing.first, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_mhd_coupled.jl

Line 75 in d942046

boundary_conditions2 = (x_neg = BoundaryConditionCoupled(1, (Indexing.last, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/examples/structured_2d_dgsem/elixir_mhd_coupled.jl

Line 77 in d942046

x_pos = BoundaryConditionCoupled(1, (Indexing.first, Indexing.i_forward), Float64,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/callbacks_step/analysis.jl

Line 624 in d942046

l2_error, linf_error = calc_error_norms(u_ode_cpu, sol.t[end], analyzer, semi_cpu,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/semidiscretization/semidiscretization.jl

Line 109 in d942046

if !(typeof(semi) <: SemidiscretizationHyperbolic) && !(typeof(semi.mesh) <: P4estMesh)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/semidiscretization/semidiscretization_hyperbolic.jl

Line 40 in d942046

performance_counter::PerformanceCounter) where {Mesh, Equations,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/semidiscretization/semidiscretization_hyperbolic.jl

Line 45 in d942046

Cache}

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/semidiscretization/semidiscretization_hyperbolic.jl

Line 86 in d942046

cache, performance_counter)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/semidiscretization/semidiscretization_hyperbolic.jl

Line 129 in d942046

cache, performance_counter)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dg.jl

Line 764 in d942046

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 161 in d942046

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Lines 308 to 309 in d942046

	neighbor_ids = unsafe_wrap_or_alloc(to, _neighbor_ids, size(interfaces.neighbor_ids))
	node_indices = unsafe_wrap_or_alloc(to, _node_indices, size(interfaces.node_indices))

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 452 in d942046

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 597 in d942046

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 97 in d942046

interfaces.node_indices[side, interface_id] = (Indexing.first, surface_index1,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 101 in d942046

interfaces.node_indices[side, interface_id] = (Indexing.last, surface_index1,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 105 in d942046

interfaces.node_indices[side, interface_id] = (surface_index1, Indexing.first,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 109 in d942046

interfaces.node_indices[side, interface_id] = (surface_index1, Indexing.last,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 114 in d942046

surface_index2, Indexing.first)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 118 in d942046

surface_index2, Indexing.last)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 130 in d942046

boundaries.node_indices[boundary_id] = (Indexing.first, Indexing.i_forward, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 133 in d942046

boundaries.node_indices[boundary_id] = (Indexing.last, Indexing.i_forward, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 136 in d942046

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.first, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 139 in d942046

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.last, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 142 in d942046

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.j_forward, Indexing.first)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 145 in d942046

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.j_forward, Indexing.last)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_parallel.jl

Line 204 in d942046

typeof(_u),

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d.jl

Line 457 in d942046

_prolong2mortars!(backend, cache, u, mesh, equations, mortar_l2, surface_integral, dg)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d.jl

Lines 801 to 802 in d942046

	_calc_surface_integral!(backend, du, u, mesh, equations, surface_integral, dg, cache)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 14 in d942046

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 264 in d942046

@kernel function interface_flux_kernel!(surface_flux_values, equations, surface_flux, nodes,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 353 in d942046

@kernel function prolong2boundaries_kernel!(u_boundaries, neighbor_ids, _node_indices, u,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 504 in d942046

boundary_interp_factor_1, boundary_interp_factor_2,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 512 to 513 in d942046

	surface_flux_values[v, l, m, 1, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 517 to 519 in d942046

	surface_flux_values[v, l, m, 2,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 523 to 524 in d942046

	surface_flux_values[v, l, m, 3, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 528 to 530 in d942046

	surface_flux_values[v, l, m, 4,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 534 to 535 in d942046

	surface_flux_values[v, l, m, 5, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 539 to 541 in d942046

	surface_flux_values[v, l, m, 6,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 591 in d942046

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 160 in d942046

@inline function _prolong2mpimortars!(backend::Backend, cache, u,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 178 in d942046

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 337 in d942046

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 452 in d942046

_calc_mpi_mortar_flux!(backend, surface_flux_values, mesh, nonconservative_terms, equations,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 457 in d942046

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_gpu_parallel.jl

Line 76 in d942046

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 8 in d942046

mutable struct P4estMPICache{BufferType <: DenseVector, VecInt <: DenseVector{<:Integer}}

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 51 in d942046

mpi_neighbor_interfaces = Adapt.adapt_structure(to, mpi_cache.mpi_neighbor_interfaces)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 64 in d942046

return P4estMPICache{BufferType, VecInt}(mpi_neighbor_ranks, mpi_neighbor_interfaces,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_structured/dg_3d.jl

Line 793 in d942046

mesh::Union{StructuredMesh{3}, P4estMesh{3}, T8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 349 in d942046

0.07601887903440395,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 356 in d942046

4.848310144356219,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	function initial_condition_taylor_green_vortex(x, t, equations::CompressibleEulerEquations3D)
	A = 1.0 # magnitude of speed
	Ms = 0.1 # maximum Mach number
	rho = 1.0
	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p + 1.0/16.0 * A^2 * rho * (cos(2*x[1])*cos(2*x[3]) + 2*cos(2*x[2]) + 2*cos(2*x[1]) + cos(2*x[2])*cos(2*x[3]))
	return prim2cons(SVector(rho, v1, v2, v3, p), equations)
	function initial_condition_taylor_green_vortex(x, t,
	equations::CompressibleEulerEquations3D)
	A = 1.0 # magnitude of speed
	Ms = 0.1 # maximum Mach number
	rho = 1.0
	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p +
	1.0 / 16.0 * A^2 * rho *
	(cos(2 * x[1]) * cos(2 * x[3]) + 2 * cos(2 * x[2]) + 2 * cos(2 * x[1]) +
	cos(2 * x[2]) * cos(2 * x[3]))
	return prim2cons(SVector(rho, v1, v2, v3, p), equations)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	solver = DGSEM(polydeg=3, surface_flux=flux_lax_friedrichs,
	volume_integral=VolumeIntegralFluxDifferencing(flux_lax_friedrichs))
	solver = DGSEM(polydeg = 3, surface_flux = flux_lax_friedrichs,
	volume_integral = VolumeIntegralFluxDifferencing(flux_lax_friedrichs))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	coordinates_max = ( 1.0, 1.0, 1.0) .* pi
	coordinates_max = (1.0, 1.0, 1.0) .* pi

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	mesh = P4estMesh(trees_per_dimension, polydeg=1,
	coordinates_min=coordinates_min, coordinates_max=coordinates_max,
	initial_refinement_level=2)
	mesh = P4estMesh(trees_per_dimension, polydeg = 1,
	coordinates_min = coordinates_min, coordinates_max = coordinates_max,
	initial_refinement_level = 2)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions_y_pos3 = BoundaryConditionCoupled(1, (Indexing.i_forward, Indexing.first), Float64,
	boundary_conditions_y_pos3 = BoundaryConditionCoupled(1,
	(Indexing.i_forward, Indexing.first),
	Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions_x_neg4 = BoundaryConditionCoupled(3, (Indexing.last, Indexing.i_forward), Float64,
	boundary_conditions_x_neg4 = BoundaryConditionCoupled(3,
	(Indexing.last, Indexing.i_forward),
	Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions_x_pos4 = BoundaryConditionCoupled(3, (Indexing.first, Indexing.i_forward), Float64,
	boundary_conditions_x_pos4 = BoundaryConditionCoupled(3,
	(Indexing.first, Indexing.i_forward),
	Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions_y_neg4 = BoundaryConditionCoupled(2, (Indexing.i_forward, Indexing.last), Float64,
	boundary_conditions_y_neg4 = BoundaryConditionCoupled(2,
	(Indexing.i_forward, Indexing.last),
	Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions_y_pos4 = BoundaryConditionCoupled(2, (Indexing.i_forward, Indexing.first), Float64,
	boundary_conditions_y_pos4 = BoundaryConditionCoupled(2,
	(Indexing.i_forward, Indexing.first),
	Float64,

[github-actions]

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

JuliaFormatter

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 161 in 9c857e4

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Lines 308 to 309 in 9c857e4

	neighbor_ids = unsafe_wrap_or_alloc(to, _neighbor_ids, size(interfaces.neighbor_ids))
	node_indices = unsafe_wrap_or_alloc(to, _node_indices, size(interfaces.node_indices))

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 452 in 9c857e4

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers.jl

Line 597 in 9c857e4

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 97 in 9c857e4

interfaces.node_indices[side, interface_id] = (Indexing.first, surface_index1,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 101 in 9c857e4

interfaces.node_indices[side, interface_id] = (Indexing.last, surface_index1,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 105 in 9c857e4

interfaces.node_indices[side, interface_id] = (surface_index1, Indexing.first,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 109 in 9c857e4

interfaces.node_indices[side, interface_id] = (surface_index1, Indexing.last,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 114 in 9c857e4

surface_index2, Indexing.first)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 118 in 9c857e4

surface_index2, Indexing.last)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 130 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.first, Indexing.i_forward, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 133 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.last, Indexing.i_forward, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 136 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.first, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 139 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.last, Indexing.j_forward)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 142 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.j_forward, Indexing.first)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_3d.jl

Line 145 in 9c857e4

boundaries.node_indices[boundary_id] = (Indexing.i_forward, Indexing.j_forward, Indexing.last)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/containers_parallel.jl

Line 204 in 9c857e4

typeof(_u),

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d.jl

Line 457 in 9c857e4

_prolong2mortars!(backend, cache, u, mesh, equations, mortar_l2, surface_integral, dg)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d.jl

Lines 801 to 802 in 9c857e4

	_calc_surface_integral!(backend, du, u, mesh, equations, surface_integral, dg, cache)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 14 in 9c857e4

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 264 in 9c857e4

@kernel function interface_flux_kernel!(surface_flux_values, equations, surface_flux, nodes,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 353 in 9c857e4

@kernel function prolong2boundaries_kernel!(u_boundaries, neighbor_ids, _node_indices, u,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 504 in 9c857e4

boundary_interp_factor_1, boundary_interp_factor_2,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 512 to 513 in 9c857e4

	surface_flux_values[v, l, m, 1, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 517 to 519 in 9c857e4

	surface_flux_values[v, l, m, 2,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 523 to 524 in 9c857e4

	surface_flux_values[v, l, m, 3, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 528 to 530 in 9c857e4

	surface_flux_values[v, l, m, 4,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 534 to 535 in 9c857e4

	surface_flux_values[v, l, m, 5, element] *
	boundary_interp_factor_1)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Lines 539 to 541 in 9c857e4

	surface_flux_values[v, l, m, 6,
	element] *
	boundary_interp_factor_2)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu.jl

Line 591 in 9c857e4

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 160 in 9c857e4

@inline function _prolong2mpimortars!(backend::Backend, cache, u,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 178 in 9c857e4

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 337 in 9c857e4

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 452 in 9c857e4

_calc_mpi_mortar_flux!(backend, surface_flux_values, mesh, nonconservative_terms, equations,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 457 in 9c857e4

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_gpu_parallel.jl

Line 76 in 9c857e4

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 8 in 9c857e4

mutable struct P4estMPICache{BufferType <: DenseVector, VecInt <: DenseVector{<:Integer}}

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 51 in 9c857e4

mpi_neighbor_interfaces = Adapt.adapt_structure(to, mpi_cache.mpi_neighbor_interfaces)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 64 in 9c857e4

return P4estMPICache{BufferType, VecInt}(mpi_neighbor_ranks, mpi_neighbor_interfaces,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_structured/dg_3d.jl

Line 793 in 9c857e4

mesh::Union{StructuredMesh{3}, P4estMesh{3}, T8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 349 in 9c857e4

0.07601887903440395,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 356 in 9c857e4

4.848310144356219,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	A = 1.0 # magnitude of speed
	A = 1.0 # magnitude of speed

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p + 1.0/16.0 * A^2 * rho * (cos(2*x[1])*cos(2*x[3]) +
	2*cos(2*x[2]) + 2*cos(2*x[1]) + cos(2*x[2])*cos(2*x[3]))
	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p +
	1.0 / 16.0 * A^2 * rho *
	(cos(2 * x[1]) * cos(2 * x[3]) +
	2 * cos(2 * x[2]) + 2 * cos(2 * x[1]) + cos(2 * x[2]) * cos(2 * x[3]))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	solver = DGSEM(polydeg=5, surface_flux=volume_flux,
	volume_integral=VolumeIntegralFluxDifferencing(volume_flux))
	solver = DGSEM(polydeg = 5, surface_flux = volume_flux,
	volume_integral = VolumeIntegralFluxDifferencing(volume_flux))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	coordinates_max = ( 1.0, 1.0, 1.0) .* pi
	coordinates_max = (1.0, 1.0, 1.0) .* pi

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	mesh = P4estMesh(trees_per_dimension, polydeg=1,
	coordinates_min=coordinates_min, coordinates_max=coordinates_max,
	periodicity=true, initial_refinement_level=initial_refinement_level)
	mesh = P4estMesh(trees_per_dimension, polydeg = 1,
	coordinates_min = coordinates_min, coordinates_max = coordinates_max,
	periodicity = true, initial_refinement_level = initial_refinement_level)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	x_pos = BoundaryConditionCoupled(1, (Indexing.first, Indexing.i_forward), Float64,
	x_pos = BoundaryConditionCoupled(1,
	(Indexing.first,
	Indexing.i_forward), Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions1 = (x_neg = BoundaryConditionCoupled(2, (Indexing.last, Indexing.i_forward), Float64,
	boundary_conditions1 = (x_neg = BoundaryConditionCoupled(2,
	(Indexing.last,
	Indexing.i_forward), Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	x_pos = BoundaryConditionCoupled(2, (Indexing.first, Indexing.i_forward), Float64,
	x_pos = BoundaryConditionCoupled(2,
	(Indexing.first,
	Indexing.i_forward), Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	boundary_conditions2 = (x_neg = BoundaryConditionCoupled(1, (Indexing.last, Indexing.i_forward), Float64,
	boundary_conditions2 = (x_neg = BoundaryConditionCoupled(1,
	(Indexing.last,
	Indexing.i_forward), Float64,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	x_pos = BoundaryConditionCoupled(1, (Indexing.first, Indexing.i_forward), Float64,
	x_pos = BoundaryConditionCoupled(1,
	(Indexing.first,
	Indexing.i_forward), Float64,

[github-actions]

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

JuliaFormatter

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 160 in fd5831e

@inline function _prolong2mpimortars!(backend::Backend, cache, u,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_gpu_parallel.jl

Line 178 in fd5831e

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 337 in fd5831e

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 452 in fd5831e

_calc_mpi_mortar_flux!(backend, surface_flux_values, mesh, nonconservative_terms, equations,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl

Line 457 in fd5831e

mesh::Union{ParallelP4estMesh{3}, ParallelT8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_gpu_parallel.jl

Line 76 in fd5831e

end # @muladd

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 8 in fd5831e

mutable struct P4estMPICache{BufferType <: DenseVector, VecInt <: DenseVector{<:Integer}}

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 51 in fd5831e

mpi_neighbor_interfaces = Adapt.adapt_structure(to, mpi_cache.mpi_neighbor_interfaces)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_p4est/dg_parallel.jl

Line 64 in fd5831e

return P4estMPICache{BufferType, VecInt}(mpi_neighbor_ranks, mpi_neighbor_interfaces,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/src/solvers/dgsem_structured/dg_3d.jl

Line 793 in fd5831e

mesh::Union{StructuredMesh{3}, P4estMesh{3}, T8codeMesh{3}},

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 349 in fd5831e

0.07601887903440395,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Trixi.jl/test/test_t8code_3d.jl

Line 356 in fd5831e

4.848310144356219,

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	ode = semidiscretize(semi, tspan; adapt_to=CuArray)
	ode = semidiscretize(semi, tspan; adapt_to = CuArray)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	tspan=(0.0, 1e-14))
	tspan = (0.0, 1e-14))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	maxiters=maxiters,
	initial_refinement_level=initial_refinement_level,
	run_profiler=run_profiler)
	maxiters = maxiters,
	initial_refinement_level = initial_refinement_level,
	run_profiler = run_profiler)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	println(io, key, ": ", mapreduce(x->x[key], min, gathered_metrics))
	println(io, key, ": ", mapreduce(x -> x[key], min, gathered_metrics))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	performance_callback = PerformanceDataCallback(interval=5)
	performance_callback = PerformanceDataCallback(interval = 5)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false),
	dt=1.0,
	save_everystep=false, callback=callbacks,
	maxiters=maxiters, verbose=false)
	integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false),
	dt = 1.0,
	save_everystep = false, callback = callbacks,
	maxiters = maxiters, verbose = false)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	A = 1.0 # magnitude of speed
	A = 1.0 # magnitude of speed

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p + 1.0/16.0 * A^2 * rho * (cos(2*x[1])*cos(2*x[3]) +
	2*cos(2*x[2]) + 2*cos(2*x[1]) + cos(2*x[2])*cos(2*x[3]))
	v1 = A * sin(x[1]) * cos(x[2]) * cos(x[3])
	v2 = -A * cos(x[1]) * sin(x[2]) * cos(x[3])
	v3 = 0.0
	p = (A / Ms)^2 * rho / equations.gamma # scaling to get Ms
	p = p +
	1.0 / 16.0 * A^2 * rho *
	(cos(2 * x[1]) * cos(2 * x[3]) +
	2 * cos(2 * x[2]) + 2 * cos(2 * x[1]) + cos(2 * x[2]) * cos(2 * x[3]))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	solver = DGSEM(polydeg=3, surface_flux=surface_flux,
	volume_integral=VolumeIntegralFluxDifferencing(volume_flux))
	solver = DGSEM(polydeg = 3, surface_flux = surface_flux,
	volume_integral = VolumeIntegralFluxDifferencing(volume_flux))

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	coordinates_max = ( 1.0, 1.0, 1.0) .* pi
	coordinates_max = (1.0, 1.0, 1.0) .* pi

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	mesh = P4estMesh(trees_per_dimension, polydeg=3,
	coordinates_min=coordinates_min, coordinates_max=coordinates_max,
	periodicity=true, initial_refinement_level=initial_refinement_level)
	mesh = P4estMesh(trees_per_dimension, polydeg = 3,
	coordinates_min = coordinates_min, coordinates_max = coordinates_max,
	periodicity = true, initial_refinement_level = initial_refinement_level)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	integrator = init(ode, CarpenterKennedy2N54(williamson_condition=false),
	dt=1.0,
	save_everystep=false, callback=callbacks)
	integrator = init(ode, CarpenterKennedy2N54(williamson_condition = false),
	dt = 1.0,
	save_everystep = false, callback = callbacks)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	Trixi.calc_volume_integral!(du_ref, u, mesh, Trixi.False(), equations, solver.volume_integral, solver, cache)
	Trixi.calc_volume_integral!(du_new, u, mesh, Trixi.False(), equations, solver.volume_integral, solver, cache)
	Trixi.calc_volume_integral!(du_ref, u, mesh, Trixi.False(), equations,
	solver.volume_integral, solver, cache)
	Trixi.calc_volume_integral!(du_new, u, mesh, Trixi.False(), equations,
	solver.volume_integral, solver, cache)

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	println("The experimental version is corrrect.")
	println("The experimental version is corrrect.")

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	println("There is a bug in the experimental version.")
	println("There is a bug in the experimental version.")

[github-actions]

[JuliaFormatter] _{reported by reviewdog 🐶}

Suggested change

	finalize(mesh)
	finalize(mesh)