#include <fft_grid_serial.hpp>

Inheritance diagram for q3::SerialFFTGrid:

Public Member Functions
	SerialFFTGrid ()=default

	SerialFFTGrid (const SerialFFTGrid &)=delete

	SerialFFTGrid (SerialFFTGrid &&)=default

	~SerialFFTGrid () override

int	concurrent_transform_slots () const override

void	execute_forward_on_buffers () const override

void	execute_forward_on_slot (int slot) const override
	Forward transform on slot `slot`, the slot-aware `execute_forward_on_buffers()`.

void	execute_inverse_raw_on_buffers () const override

void	execute_inverse_raw_on_slot (int slot) const override
	Inverse transform on slot `slot`, the slot-aware `execute_inverse_raw_on_buffers()`.

void	forward (Complex *data) const override

int	g_vector_to_fft_grid_index (int miller_1, int miller_2, int miller_3) const override

int	grid_dimension_1 () const override

int	grid_dimension_2 () const override

int	grid_dimension_3 () const override

void	init (int dimension_1, int dimension_2, int dimension_3, bool use_measured_planning=false) override

Complex *	input_buffer () const override

void	inverse (Complex *data) const override

void	inverse_raw (Complex *data) const override

void	inverse_raw_on_slot (Complex *data, int slot) const override

int	local_grid_points () const override

int	local_offset () const override

SerialFFTGrid &	operator= (const SerialFFTGrid &)=delete

SerialFFTGrid &	operator= (SerialFFTGrid &&)=default

Complex *	output_buffer () const override

void	prepare_forward_from_output () const override

void	prepare_forward_from_output_on_slot (int slot) const override
	Output-to-input copy on slot `slot`, the slot-aware `prepare_forward_from_output()`.

void	reduce_g_space (Complex *, int) const override

Complex *	slot_input_buffer (int slot) const override

Complex *	slot_output_buffer (int slot) const override

int	total_grid_points () const override

Public Member Functions inherited from q3::FFTGrid
virtual	~FFTGrid ()=default

virtual void	assign_sticks (const std::vector< int > &)

virtual bool	co_locates_mirrors () const

virtual void	forward_batched (Complex *data, int batch_size) const

virtual void	gather_local_g_space (const Complex full_g_space, Complex local_g_space) const

virtual int	global_real_space_index (int local_index) const

virtual void	initialize_batched_plans (int, bool=false) const

virtual int	initialized_batched_plan_size () const

virtual void	inverse_raw_batched (Complex *data, int batch_size) const

virtual int	local_g_space_points () const

virtual bool	owns_stick (int, int) const

virtual void	scatter_local_g_space (const Complex local_g_space, Complex full_g_space) const
	The reverse: place this rank's reciprocal-space points into a full array.

virtual int	slab_dimension_1 () const

Detailed Description

SerialFFTGrid — node-local 3D FFT implementation using FFTW3.

This implementation assumes the entire grid resides in the memory of a single process. It supports OpenMP multi-threading within FFTW if Q3_USE_FFTW_THREADS is defined.

Definition at line 40 of file fft_grid_serial.hpp.

Constructor & Destructor Documentation

◆ SerialFFTGrid() [1/3]

q3::SerialFFTGrid::SerialFFTGrid ( )

default

◆ ~SerialFFTGrid()

q3::SerialFFTGrid::~SerialFFTGrid ( )

inlineoverride

Definition at line 55 of file fft_grid_serial.hpp.

◆ SerialFFTGrid() [2/3]

q3::SerialFFTGrid::SerialFFTGrid ( const SerialFFTGrid & )

delete

◆ SerialFFTGrid() [3/3]

q3::SerialFFTGrid::SerialFFTGrid ( SerialFFTGrid && )

default

Member Function Documentation

◆ concurrent_transform_slots()

int q3::SerialFFTGrid::concurrent_transform_slots ( ) const

inlineoverridevirtual

How many transforms may run at once: one per set of scratch arrays init() planned. Slot 0 is the pair every other entry point already uses, so a caller that asks for one slot runs the historical code.

Reimplemented from q3::FFTGrid.

Definition at line 295 of file fft_grid_serial.hpp.

◆ execute_forward_on_buffers()

void q3::SerialFFTGrid::execute_forward_on_buffers ( ) const

inlineoverridevirtual

Execute a forward FFT on the internal buffers (no copy).

Caller must have populated input_buffer() before calling. Result is available in output_buffer() after this call.

Implements q3::FFTGrid.

Definition at line 279 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

Referenced by q3::gpu::CudaGPUFFTGrid::execute_forward_on_buffers(), q3::gpu::HIPGPUFFTGrid::execute_forward_on_buffers(), and q3::gpu::SYCLGPUFFTGrid::execute_forward_on_buffers().

◆ execute_forward_on_slot()

void q3::SerialFFTGrid::execute_forward_on_slot ( int ) const

inlineoverridevirtual

Forward transform on slot slot, the slot-aware execute_forward_on_buffers().

Reimplemented from q3::FFTGrid.

Definition at line 316 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

◆ execute_inverse_raw_on_buffers()

void q3::SerialFFTGrid::execute_inverse_raw_on_buffers ( ) const

inlineoverridevirtual

Execute a raw inverse FFT on the internal buffers (no copy, no normalization).

Caller must have populated input_buffer() before calling. Result is available in output_buffer() after this call.

Implements q3::FFTGrid.

Definition at line 274 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

Referenced by q3::gpu::CudaGPUFFTGrid::execute_inverse_raw_on_buffers(), q3::gpu::HIPGPUFFTGrid::execute_inverse_raw_on_buffers(), and q3::gpu::SYCLGPUFFTGrid::execute_inverse_raw_on_buffers().

◆ execute_inverse_raw_on_slot()

void q3::SerialFFTGrid::execute_inverse_raw_on_slot ( int ) const

inlineoverridevirtual

Inverse transform on slot slot, the slot-aware execute_inverse_raw_on_buffers().

Reimplemented from q3::FFTGrid.

Definition at line 309 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

◆ forward()

void q3::SerialFFTGrid::forward ( Complex * data ) const

inlineoverridevirtual

Forward FFT: real-space -> reciprocal-space (unnormalized).

Implements q3::FFTGrid.

Definition at line 217 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

Referenced by q3::gpu::CudaGPUFFTGrid::forward(), q3::gpu::HIPGPUFFTGrid::forward(), q3::gpu::SYCLGPUFFTGrid::forward(), q3::gpu::SYCLGPUFFTGrid::forward_single(), and q3::gpu::SYCLGPUFFTGrid::forward_transform().

◆ g_vector_to_fft_grid_index()

int q3::SerialFFTGrid::g_vector_to_fft_grid_index	(	int	miller_1,
		int	miller_2,
		int	miller_3
	)		const

inlineoverridevirtual

Map a G-vector (integer Miller indices) to FFT array index. Handles negative indices via wrapping.

Implements q3::FFTGrid.

Definition at line 246 of file fft_grid_serial.hpp.

Referenced by q3::complex_field_from_reciprocal_space(), q3::gpu::CudaGPUFFTGrid::g_vector_to_fft_grid_index(), q3::gpu::HIPGPUFFTGrid::g_vector_to_fft_grid_index(), and q3::gpu::SYCLGPUFFTGrid::g_vector_to_fft_grid_index().

◆ grid_dimension_1()

int q3::SerialFFTGrid::grid_dimension_1 ( ) const

inlineoverridevirtual

Grid size along the first dimension.

Implements q3::FFTGrid.

Definition at line 204 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::grid_dimension_1(), q3::gpu::HIPGPUFFTGrid::grid_dimension_1(), and q3::gpu::SYCLGPUFFTGrid::grid_dimension_1().

◆ grid_dimension_2()

int q3::SerialFFTGrid::grid_dimension_2 ( ) const

inlineoverridevirtual

Grid size along the second dimension.

Implements q3::FFTGrid.

Definition at line 207 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::grid_dimension_2(), q3::gpu::HIPGPUFFTGrid::grid_dimension_2(), and q3::gpu::SYCLGPUFFTGrid::grid_dimension_2().

◆ grid_dimension_3()

int q3::SerialFFTGrid::grid_dimension_3 ( ) const

inlineoverridevirtual

Grid size along the third dimension.

Implements q3::FFTGrid.

Definition at line 210 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::grid_dimension_3(), q3::gpu::HIPGPUFFTGrid::grid_dimension_3(), and q3::gpu::SYCLGPUFFTGrid::grid_dimension_3().

◆ init()

void q3::SerialFFTGrid::init	(	int	dimension_1,
		int	dimension_2,
		int	dimension_3,
		bool	use_measured_planning = `false`
	)

inlineoverridevirtual

Initialize with grid dimensions (dimension_1 x dimension_2 x dimension_3).

How many transforms callers may drive at once. One slot – the historical behaviour – when there is one thread to run them on, so a single-threaded run allocates nothing extra and executes the same plan on the same buffers it always did.

Each additional slot costs two more grids of scratch. That is small beside the wavefunction storage a grid this size implies, and it is what buys a band loop that scales. The alternative – FFTW's own intra-transform threading, which is what this class used to do with OMP_NUM_THREADS – pays a fork and join inside every one of the thousands of transforms a Davidson iteration issues, and the measurements in docs/performance_nersc_perlmutter.md show it bottoming out at eight threads and then getting worse than serial. That document names threading across bands with serial transforms as the scalable axis; this is it.

A ceiling on what the extra slots may cost, because the thread count is the machine's and the grid size is the calculation's, and their product is nobody's decision. Sixty-four threads on a 128^3 grid would otherwise claim four gibibytes of scratch that no input asked for. Past this point the slots stop being added and the band loop runs on fewer threads than the machine has – which costs some parallelism, and is the right way round: a slower run is recoverable and an out-of-memory run is not.

Out-of-place plans: input from work_, output to output_buffer_. Out-of-place avoids internal data shuffling and is typically faster.

Slots 1..N-1. Each gets its own plan over its own arrays rather than borrowing slot 0's through fftw_execute_dft, because that shortcut is only valid when the new arrays share the plan's alignment class – a property of the allocator, not of anything checked here. Planning is serial by construction: FFTW's planner is not re-entrant, and this runs once at setup, before any parallel region.

Implements q3::FFTGrid.

Definition at line 74 of file fft_grid_serial.hpp.

References q3::execution::threading::maximum_threads().

Referenced by q3::complex_field_from_reciprocal_space(), q3::gpu::CudaGPUFFTGrid::init(), q3::gpu::HIPGPUFFTGrid::init(), and q3::gpu::SYCLGPUFFTGrid::init().

◆ input_buffer()

Complex * q3::SerialFFTGrid::input_buffer ( ) const

inlineoverridevirtual

Access the FFT grid's internal input buffer for zero-copy transforms.

Callers scatter G-space data directly into this buffer, then call execute_inverse_raw_on_buffers() to transform in-place. This eliminates the memcpy overhead of inverse_raw() / forward().

The buffer has at least local_grid_points() elements of usable space.

Implements q3::FFTGrid.

Definition at line 266 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::input_buffer(), q3::gpu::HIPGPUFFTGrid::input_buffer(), and q3::gpu::SYCLGPUFFTGrid::input_buffer().

◆ inverse()

void q3::SerialFFTGrid::inverse ( Complex * data ) const

inlineoverridevirtual

Inverse FFT: reciprocal-space -> real-space (includes 1/N normalization).

Implements q3::FFTGrid.

Definition at line 226 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

Referenced by q3::gpu::CudaGPUFFTGrid::inverse(), q3::gpu::HIPGPUFFTGrid::inverse(), and q3::gpu::SYCLGPUFFTGrid::inverse().

◆ inverse_raw()

void q3::SerialFFTGrid::inverse_raw ( Complex * data ) const

inlineoverridevirtual

Inverse FFT without 1/N normalization (raw FFTW backward).

Implements q3::FFTGrid.

Definition at line 237 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, and q3::ProfilingCounters::instance().

Referenced by q3::complex_field_from_reciprocal_space(), q3::gpu::CudaGPUFFTGrid::inverse_raw(), q3::gpu::HIPGPUFFTGrid::inverse_raw(), q3::gpu::SYCLGPUFFTGrid::inverse_raw(), q3::gpu::SYCLGPUFFTGrid::inverse_raw_single(), and q3::gpu::SYCLGPUFFTGrid::inverse_transform().

◆ inverse_raw_on_slot()

void q3::SerialFFTGrid::inverse_raw_on_slot	(	Complex *	data,
		int	slot
	)		const

inlineoverridevirtual

The same copy-in, transform, copy-out inverse_raw() performs, driven through slot slot's scratch and plan. Slot 0 is inverse_raw() itself.

Reimplemented from q3::FFTGrid.

Definition at line 327 of file fft_grid_serial.hpp.

References q3::ProfilingCounters::fft_call_count, q3::ProfilingCounters::instance(), slot_input_buffer(), and slot_output_buffer().

◆ local_grid_points()

int q3::SerialFFTGrid::local_grid_points ( ) const

inlineoverridevirtual

Number of real-space grid points stored on the local rank.

Implements q3::FFTGrid.

Definition at line 254 of file fft_grid_serial.hpp.

References total_grid_points().

Referenced by q3::gpu::CudaGPUFFTGrid::local_grid_points(), q3::gpu::HIPGPUFFTGrid::local_grid_points(), and q3::gpu::SYCLGPUFFTGrid::local_grid_points().

◆ local_offset()

int q3::SerialFFTGrid::local_offset ( ) const

inlineoverridevirtual

Offset into the global grid for the local segment (e.g., start of slab).

Implements q3::FFTGrid.

Definition at line 258 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::local_offset(), q3::gpu::HIPGPUFFTGrid::local_offset(), and q3::gpu::SYCLGPUFFTGrid::local_offset().

◆ operator=() [1/2]

SerialFFTGrid & q3::SerialFFTGrid::operator= ( const SerialFFTGrid & )

delete

◆ operator=() [2/2]

SerialFFTGrid & q3::SerialFFTGrid::operator= ( SerialFFTGrid && )

default

◆ output_buffer()

Complex * q3::SerialFFTGrid::output_buffer ( ) const

inlineoverridevirtual

Access the FFT grid's internal output buffer for zero-copy transforms.

After execute_inverse_raw_on_buffers(), real-space data is here. After execute_forward_on_buffers(), G-space data is here.

For in-place plans (MPI FFT), this returns the same pointer as input_buffer(). For out-of-place plans (Serial FFT), this is a separate buffer.

Implements q3::FFTGrid.

Definition at line 270 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::output_buffer(), q3::gpu::HIPGPUFFTGrid::output_buffer(), and q3::gpu::SYCLGPUFFTGrid::output_buffer().

◆ prepare_forward_from_output()

void q3::SerialFFTGrid::prepare_forward_from_output ( ) const

inlineoverridevirtual

Prepare the input buffer for a forward transform after modifying the output buffer.

For out-of-place plans (Serial FFT), this copies output_buffer → input_buffer. For in-place plans (MPI FFT), this is a no-op (same buffer).

Call this between the real-space multiply and execute_forward_on_buffers().

Implements q3::FFTGrid.

Definition at line 284 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::prepare_forward_from_output(), q3::gpu::HIPGPUFFTGrid::prepare_forward_from_output(), and q3::gpu::SYCLGPUFFTGrid::prepare_forward_from_output().

◆ prepare_forward_from_output_on_slot()

void q3::SerialFFTGrid::prepare_forward_from_output_on_slot ( int ) const

inlineoverridevirtual

Output-to-input copy on slot slot, the slot-aware prepare_forward_from_output().

Reimplemented from q3::FFTGrid.

Definition at line 338 of file fft_grid_serial.hpp.

References slot_input_buffer(), and slot_output_buffer().

◆ reduce_g_space()

void q3::SerialFFTGrid::reduce_g_space	(	Complex *	data,
		int	number_of_elements
	)		const

inlineoverridevirtual

Reduce G-space data across the pool to make it redundant on all ranks.

Serial: no-op. MPI: MPI_Allreduce(MPI_IN_PLACE, ..., MPI_SUM, pool_comm).

Implements q3::FFTGrid.

Definition at line 262 of file fft_grid_serial.hpp.

Referenced by q3::gpu::CudaGPUFFTGrid::reduce_g_space(), q3::gpu::HIPGPUFFTGrid::reduce_g_space(), and q3::gpu::SYCLGPUFFTGrid::reduce_g_space().

◆ slot_input_buffer()

Complex * q3::SerialFFTGrid::slot_input_buffer ( int ) const

inlineoverridevirtual

The input buffer of transform slot slot, the slot-aware form of input_buffer().

Reimplemented from q3::FFTGrid.

Definition at line 299 of file fft_grid_serial.hpp.

Referenced by inverse_raw_on_slot(), and prepare_forward_from_output_on_slot().

◆ slot_output_buffer()

Complex * q3::SerialFFTGrid::slot_output_buffer ( int ) const

inlineoverridevirtual

The output buffer of transform slot slot, the slot-aware form of output_buffer().

Reimplemented from q3::FFTGrid.

Definition at line 304 of file fft_grid_serial.hpp.

Referenced by inverse_raw_on_slot(), and prepare_forward_from_output_on_slot().

◆ total_grid_points()

int q3::SerialFFTGrid::total_grid_points ( ) const

inlineoverridevirtual

Total number of real-space grid points across all processes.

Implements q3::FFTGrid.

Definition at line 213 of file fft_grid_serial.hpp.

Referenced by q3::complex_field_from_reciprocal_space(), q3::gpu::CudaGPUFFTGrid::init(), q3::gpu::HIPGPUFFTGrid::init(), local_grid_points(), q3::gpu::CudaGPUFFTGrid::total_grid_points(), q3::gpu::HIPGPUFFTGrid::total_grid_points(), and q3::gpu::SYCLGPUFFTGrid::total_grid_points().

The documentation for this class was generated from the following file:

src/layer_3_representation/basis/planewave/internal/fft/fft_grid_serial.hpp

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ SerialFFTGrid() [1/3]

◆ ~SerialFFTGrid()

◆ SerialFFTGrid() [2/3]

◆ SerialFFTGrid() [3/3]

Member Function Documentation

◆ concurrent_transform_slots()

◆ execute_forward_on_buffers()

◆ execute_forward_on_slot()

◆ execute_inverse_raw_on_buffers()

◆ execute_inverse_raw_on_slot()

◆ forward()

◆ g_vector_to_fft_grid_index()

◆ grid_dimension_1()

◆ grid_dimension_2()

◆ grid_dimension_3()

◆ init()

◆ input_buffer()

◆ inverse()

◆ inverse_raw()

◆ inverse_raw_on_slot()

◆ local_grid_points()

◆ local_offset()

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ output_buffer()

◆ prepare_forward_from_output()

◆ prepare_forward_from_output_on_slot()

◆ reduce_g_space()

◆ slot_input_buffer()

◆ slot_output_buffer()

◆ total_grid_points()