Essentials of running GTC
Bashrc | Job-Script | Makefile - To run GTC(v4.6U) in ParamPravega and PTG
Makefile_PTG
Go to the documentation of this file.
1 # This file is part of GTC version 4.6
2 # GTC version 4.6 is released under the 3-Clause BSD license:
3 
4 # Copyright (c) 2002,2010,2016, GTC Team (team leader: Zhihong Lin, zhihongl@uci.edu)
5 # All rights reserved.
6 
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are met:
9 
10 # 1. Redistributions of source code must retain the above copyright notice,
11 # this list of conditions and the following disclaimer.
12 
13 # 2. Redistributions in binary form must reproduce the above copyright notice,
14 # this list of conditions and the following disclaimer in the documentation
15 # and/or other materials provided with the distribution.
16 
17 # 3. Neither the name of the GTC Team nor the names of its contributors may be
18 # used to endorse or promote products derived from this software without
19 # specific prior written permission.
20 # ==============================================================================
21 
22 #key parameters:
23 OPENMP=y
24 DOUBLE_PRECISION=y
25 ADIOS2=n
26 DEBUG=n
27 # set SDP_OUT to y to enable synthetic diagnostic platform outputs
28 SDP_OUT=n
29 COMPILER=default
30 #ICONFIG=TOKAMAK
31 #ICONFIG=CYLINDER
32 ICONFIG=TOROIDAL3D
33 
34 # GPU and field solver flags
35 NETCDF=y
36 GPU=n
37 GPU_UM=n
38 PETSc=y
39 AMGX=n
40 HYPRE=n
41 HYPRE_USES_GPU=n
42 TIME_MPI=y
43 NVTX=n
44 CUDA_MPI=n
45 ############################################################################
46 # Makefile to build the GTC code
47 # ==================================
48 #
49 # You only need to type "gmake" to build the code on the platforms
50 # defined below. The makefile runs the "uname -s" command to detect
51 # the operating system automatically. Other options are:
52 #
53 # % gmake OPENMP=y Builds the code with OpenMP support
54 # % gmake OPENMP=n Builds the code WITHOUT OpenMP support
55 # The default is with OpenMP support
56 #
57 # % gmake DOUBLE_PRECISION=y Builds with 8-byte floating point precision
58 # The default is single precision
59 #
60 # % gmake PETSc=y Builds the code using the PETSc parallel matrix
61 # solving library. This is necessary to run the
62 # electromagnetic version of GTC.
63 #
64 # % gmake DEBUG=y Compiles the files with debugging flags.
65 # The default is no debug option.
66 #
67 # If COMPILER is set to 'default' the Makefile will select the appropriate
68 # compiler for the target. If the user wished to override the target
69 # machine default, COMPILER may be set to 'portland', 'intel', or 'gnu'.
70 #
71 # Special targets:
72 #
73 # % gmake clean Removes the executable and all object files (*.o)
74 #
75 # % gmake cleanomp Removes the executable and the object files
76 # containing OpenMP directives
77 #
78 # % gmake doc Rebuilds the documentation.
79 #
80 #############################################################################
81 # Default executable name
82 CMD := gtc
83 ALL:${CMD}
84 LIB :=
85 LD_LIB :=
86 
87 ##### inquire the hostname to judge the system name, to determine what target we're compiling on
88 ifneq (,${NERSC_HOST})
89  SYSTEMS := ${NERSC_HOST}
90 else ifneq (,${HOST})
91  SYSTEMS := ${HOST}
92 else
93  ifneq (,$(findstring ppc64le,$(shell uname -m)))
94  SYSTEMS := ppc64le
95  else
96  SYSTEMS := $(shell hostname)
97  endif
98 endif
99 
100 ##### target machine determines the default compiler, and paths to libraries #####
101 # theta
102 ifneq (,$(findstring thetalogin,$(SYSTEMS)))
103  SYSTEM_FOUND=y
104  ifeq ($(COMPILER),default)
105  COMPILER := cray
106  endif
107  CMP := ftn
108  CCMP := cc
109 # module swap PrgEnv-intel PrgEnv-cray
110 # module load cray-petsc/3.12.4.1 cray-netcdf-hdf5parallel/4.7.3.3
111 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
112  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
113 # PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
114  NETCDF_LIB = -I${NETCDF_DIR}/include
115  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
116  ifeq ($(HYPRE),y)
117  HYPRE_LIB = -I${HYPRE_ROOT}/src/hypre/include
118  HYPRE_LD_LIB = -L${HYPRE_ROOT}/src/hypre/lib -lHYPRE
119  endif
120 endif
121 
122 # thetagpu
123 ifneq (,$(findstring thetagpu,$(SYSTEMS)))
124  SYSTEM_FOUND=y
125  ifeq ($(COMPILER),default)
126  COMPILER := portland
127  endif
128  CMP := mpif90
129  CCMP := mpicc
130  NVCC := nvcc
131 # module swap PrgEnv-intel PrgEnv-cray
132 # module load cray-petsc/3.12.4.1 cray-netcdf-hdf5parallel/4.7.3.3
133 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
134  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
135 # PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
136  NETCDF_LIB = -I${NETCDF_DIR}/include
137  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf ${NETCDF_DIR}/lib/libnetcdff.a
138  CUDA_LIB := -lstdc++ -L/soft/hpc-sdk/Linux_x86_64/21.7/math_libs/lib64 -L/soft/hpc-sdk/Linux_x86_64/21.7/cuda/lib64 -lcudart -lcufft -lnvToolsExt
139  ifeq ($(HYPRE),y)
140  HYPRE_LIB = -I${HYPRE_ROOT}/src/hypre/include
141  HYPRE_LD_LIB = -L${HYPRE_ROOT}/src/hypre/lib -lHYPRE
142  endif
143 endif
144 # cori
145 ifneq (,$(findstring cori,$(SYSTEMS)))
146  ifeq ($(COMPILER),default)
147  COMPILER := intel
148  endif
149  CMP := ftn
150  CCMP := cc
151 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
152  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
153  ifeq ($(NETCDF),y)
154  NETCDF_LIB = -I${NETCDF_DIR}/include
155  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
156  endif
157  LIBD = /global/u2/x/xiao/AdioFranklin/lib
158  INCD = /global/u2/x/xiao/AdioFranklin/include
159  ADIOS2_DIR = /project/projectdirs/m808/kmehta/adios2-12.16.18/installation/intel/18.0.1
160  ADIOS2_LIBS = -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet -ladios2_atl -lstdc++
161  ifeq ($(HYPRE),y)
162  HYPRE_LD_LIB = -L${HYPRE_ROOT}/lib -lHYPRE
163  HYPRE_LIB = -I${HYPRE_INC}
164  endif
165 endif
166 
167 #perlmutter
168 ifneq (,$(findstring perlmutter,$(SYSTEMS)))
169  ifeq ($(COMPILER),default)
170  COMPILER := portland
171  DEBUG_OPT_ := -g -C -gopt -Mchkstk -Mcoff -Mdwarf3 -Melf -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
172  endif
173 
174  ifeq ($(GPU),y)
175  COMPILER := portland
176  NVCC := CC -cuda
177  CUDA_LIB := -lstdc++ -lcudart -lcufft -cuda -rpath=/opt/cray/libfabric/1.11.0.4.75/lib64
178  DEBUG_OPT_ := -g -C -gopt -Mchkstk -Mcoff -Mdwarf3 -Melf -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
179  endif
180  CMP := ftn
181  CCMP := cc
182  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
183  ifeq ($(NETCDF),y)
184  NETCDF_LIB = -I${NETCDF_DIR}/include
185  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
186  endif
187  LIBD = /global/u2/x/xiao/AdioFranklin/lib
188  INCD = /global/u2/x/xiao/AdioFranklin/include
189  ADIOS2_DIR = /project/projectdirs/m808/kmehta/adios2-12.16.18/installation/intel/18.0.1
190  ADIOS2_LIBS = -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet -ladios2_atl -lstdc++
191  ifeq ($(HYPRE),y)
192  HYPRE_LIB = -I${HYPRE_INC}
193  HYPRE_LD_LIB = ${HYPRE_LFLAGS}
194  endif
195 endif
196 
197 # summit
198 #ifneq (,$(findstring summit,$(SYSTEMS))) # can not work now
199 #ifneq (,$(findstring ppc64le,$(shell uname -m)))
200 ifneq (,$(findstring ppc64le,$(SYSTEMS)))
201  ifeq ($(OLCF_AMGX_ROOT),)
202  AMGX_ROOT = /ccs/proj/csc190/gtc/amgx/summit
203  else
204  AMGX_ROOT = $(OLCF_AMGX_ROOT)
205  endif
206  ifeq ($(OLCF_HYPRE_ROOT),)
207  ifeq ($(HYPRE_USES_GPU),y)
208  HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre_gpu/summit
209  else
210  HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre/summit
211  endif
212  else
213  #ifeq ($(HYPRE_USES_GPU),y)
214  # $(error To use GPU-enabled Hypre, please unload the hypre module and use the custom hypre build)
215  #endif
216  HYPRE_ROOT = $(OLCF_HYPRE_ROOT)
217  endif
218  CMP := mpif90
219  CCMP := mpicc
220  FCMP := gfortran
221  ifeq ($(COMPILER),default)
222  COMPILER := portland
223  ifeq ($(GPU), y)
224  CUDA_LIB := -lstdc++ -L$(dir $(shell which nvcc))../lib64 -lcudart -lcufft -lnvToolsExt
225  NVCC := nvcc
226  endif
227  DEBUG_OPT_ := -g -C -gopt -Mchkfpstk -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
228  FCMP := pgf90
229  endif
230  ifeq ($(PETSc),y)
231  PETSC_LIB = -I${OLCF_PETSC_ROOT}/include
232  PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
233  endif
234  ifeq ($(NETCDF),y)
235  NETCDF_LIB = -I${OLCF_NETCDF_FORTRAN_ROOT}/include
236  NETCDF_LD_LIB = -L${OLCF_NETCDF_FORTRAN_ROOT}/lib -R${OLCF_NETCDF_FORTRAN_ROOT}/lib -lnetcdff -L${OLCF_CUDA_ROOT}/lib64/ -lnvToolsExt
237  endif
238  HDF5_LIB = -I${OLCF_HDF5_ROOT}/include
239  HDF5_LD_LIB = -L${OLCF_HDF5_ROOT}/lib -R${OLCF_HDF5_ROOT}/lib -lhdf5_fortran
240  MPI_LIB := -I${OLCF_OMPI_ROOT}/include
241  CUDA_OPT := -I$(dir $(shell which mpif90))../include
242  ifeq ($(AMGX),y)
243  AMGX_LIB = -I${AMGX_ROOT}/../AMGX/base/include
244  AMGX_LD_LIB = -L${AMGX_ROOT}/lib -R${AMGX_ROOT}/lib -lamgxsh -L$(CUDA_DIR)/targets/ppc64le-linux/lib -R$(CUDA_DIR)/targets/ppc64le-linux/lib -lcusparse
245  endif
246  ifeq ($(HYPRE),y)
247  HYPRE_LIB = -I${HYPRE_ROOT}/hypre/src/hypre//include -I${HYPRE_ROOT}/include
248  HYPRE_LD_LIB = -L${HYPRE_ROOT}/hypre/src/hypre/lib -R${HYPRE_ROOT}/hypre/src/hypre/lib -L${HYPRE_ROOT}/lib -R${HYPRE_ROOT}/lib -lHYPRE
249  ifeq ($(HYPRE_USES_GPU),y)
250  HYPRE_LD_LIB += -L${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -R${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -lnvToolsExt
251  endif
252  endif
253 # ADIOS2_DIR=/ccs/proj/phy130/wenhao/adios2-2.6.0/installation
254  ADIOS2_DIR=/gpfs/alpine/world-shared/csc143/kmehta/ADIOS2-2.8.3/installation/pgi/20.4
255  #ADIOS2_DIR=/gpfs/alpine/world-shared/csc143/kmehta/ADIOS2-2.5.0/installation/pgi/19.9/
256  #ADIOS2_DIR=${OLCF_ADIOS2_ROOT}
257  ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)
258 endif
259 
260 # power
261 ifneq (,$(findstring power,$(SYSTEMS)))
262  COMPILER := portland
263  CMP :=mpif90
264 # PETSC_OPT = -D_PETSc35BEFORE##only needed for kink/tearing
265  PETSC_LIB = -I${PETSC_DIR}/include
266  PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
267  CUDA_OPT := -I$(dir $(shell which mpif90))../include
268 # NETCDF_LIB = -I${NETCDF_DIR}/include
269 # NETCDF_LD_LIB = -L${NETCDF_DIR}/lib
270 # LIBD = /ccs/home/hardes/gtc_adios/jaguarlib
271 # INCD = /ccs/home/hardes/gtc_adios/jaguarinclude
272 # ADIOS_LIB = -I$(INCD)
273 # ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
274 endif
275 
276 # Tianhe-1A
277 ifneq (,$(findstring ln,$(SYSTEMS)))
278  ifeq ($(COMPILER),default)
279  ifeq ($(GPU),y)
280  COMPILER := portland
281  else
282  COMPILER := intel
283  endif
284  endif
285  CMP := mpif90
286 
287  PETSC_solver = petsc.o
288  PETSC_OPT = -D_PETSc31P8ANDBEFORE
289  #PETSC_OPT += -D_PETSc30ANDBEFORE
290  ifneq (,${PETSC_DIR})
291  include ${PETSC_DIR}/conf/variables
292  endif
293 
294  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
295  PETSC_LD_LIB := ${PETSC_KSP_LIB}
296 
297  NETCDF_LIB := -I${NETCDF_DIR}/include
298  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
299  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
300 endif
301 
302 # PKU Weiming, cluster
303 ifneq (,$(findstring pku,$(SYSTEMS)))
304  ifeq ($(COMPILER),default)
305  COMPILER := intel
306  endif
307  CMP := mpiifort
308 
309  #PETSC_solver = petsc.o
310  #PETSC_OPT = -D_PETSc31P8ANDBEFORE
311  #PETSC_OPT += -D_PETSc30ANDBEFORE
312  ifneq (,${PETSC_DIR})
313  # include ${PETSC_DIR}/conf/variables
314  include ${PETSC_DIR}/lib/petsc/conf/variables
315  endif
316 
317  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
318  PETSC_LD_LIB := ${PETSC_KSP_LIB}
319 
320  NETCDF_LIB := -I${NETCDF_DIR}/include
321  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff \
322  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
323 endif
324 
325 #sun
326 ifneq (,$(findstring sun,$(SYSTEMS)))
327  ifeq ($(COMPILER),default)
328  COMPILER := gnu
329  endif
330  CMP := mpif90
331  CCMP := mpicc
332  HDF5_INCLUDE := -I/opt/hdf5_parallel/include
333 # PETSC_INCLUDE := -I${PETSC_DIR}/include -I${PETSC_DIR}/${PETSC_ARCH}/include -I${PETSC_DIR}/include/petsc/finclude
334  HDF5_LIB := -lhdf5_fortran -L/opt/hdf5_parallel/lib
335  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/petsc/finclude -I${PETSC_DIR}/${PETSC_ARCH}/include
336  PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
337 
338 # ADIOS2_DIR=/home/kmehta/ADIOS2-2.6.0/installation/
339 # ADIOS2_DIR=/home/wenhao/Downloads/adios2-2.6.0/installation
340 # ADIOS2_DIR=/opt/adios-2.6.0/installation
341  ADIOS2_DIR=/store/home/wenhao/Downloads/adios2-2.8.3/installation
342  ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)
343  ifeq ($(HYPRE),y)
344  HYPRE_LD_LIB = -L${HYPRE_LD} -lHYPRE
345  HYPRE_LIB = -I${HYPRE_INC}
346  endif
347  ifeq ($(NETCDF),y)
348  NETCDF_DIR = /usr
349  NETCDF_LIB = -I${NETCDF_DIR}/include/
350  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib/ -lnetcdff
351  endif
352 endif
353 
354 #iop
355 ifneq (,$(findstring indac,$(SYSTEMS)))
356  ifeq ($(COMPILER),default)
357  COMPILER := intel
358  endif
359  CMP := mpif90
360 
361  #PETSC_solver = petsc.o
362  PETSC_OPT = -D_PETSc31P8ANDBEFORE
363  #PETSC_OPT += -D_PETSc30ANDBEFORE
364  ifneq (,${PETSC_DIR})
365  include ${PETSC_DIR}/conf/variables
366  endif
367 
368  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
369  PETSC_LD_LIB := ${PETSC_KSP_LIB}
370  ifeq ($(NETCDF),y)
371  NETCDF_LIB := -I${NETCDF_DIR}/include
372  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
373  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
374  endif
375 endif
376 
377 #### compiler determines debugging, openmp, and double precision flags #####
378 # intel fortran
379 ifeq ($(COMPILER),intel)
380  OPTIMOPT := -O
381  #OPTIMOPT := -O3
382  #OPTIMOPT += -fp-model fast=2
383  #OPTIMOPT += -align array64byte
384  #OPTIMOPT += -no-vec
385 # debug flags changed by L. Shi to suppress long warning outputs.
386  #DEBUGOPT := -g -free -pc64 -check all -warn nounused -warn all -debug all -debug-parameters all -fp-stack-check
387  DEBUGOPT := -g -free -pc64 -check 'all,noarg_temp_created' -warn none -debug all -debug-parameters all -fp-stack-check
388  DEBUGCOPT := -O0 -g
389  OMPOPT := -qopenmp
390  DPOPT := -fpconstant
391 endif
392 
393 # portland group fortran
394 ifeq ($(COMPILER),portland)
395  OPTIMOPT := -fast -Mfree -Kieee
396  #DEBUGOPT := -g -C -gopt -Mbounds -Mchkfpstk -Mchkptr -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -traceback -Minform=inform
397  # -Mchkptr is currently not working well with GPU compiling, so temporarily opted out
398  DEBUGOPT := $(DEBUG_OPT_)
399  DEBUGCOPT := -O0 -g
400  OMPOPT := -mp
401  DPOPT := -DDOUBLE_PRECISION
402 endif
403 
404 ### assemble all the options
405 OPT :=
406 LIB :=
407 PREPROCESS_OPT := -E
408 FCMP ?= $(CMP)
409 
410 # GNU gfortran
411 ifeq ($(COMPILER),gnu)
412  OPT += -ffree-line-length-0 -Wuninitialized
413  OPTIMOPT := -O
414  DEBUGOPT := -g -fcheck=all,no-array-temps,no-bounds
415  DEBUGCOPT := -O0 -g -Wall
416 # DEBUGOPT := -g -fcheck=all,no-array-temps
417  OMPOPT := -fopenmp
418  DPOPT :=
419 endif
420 
421 # ALCF Theta Cray ftn (KNL, no OpenACC)
422 ifeq ($(COMPILER),cray)
423  # no -N0 option comparable to -ffree-line-length-0
424  OPT += -xacc -h noacc -N1023 -O shortcircuit2 -Werror
425  OPTIMOPT := -O1
426  DEBUGOPT := -g -nobounds
427  DEBUGCOPT := -O0 -g -Wall
428  OMPOPT := -homp
429  DPOPT :=
430  PREPROCESS_OPT := -eP
431 endif
432 
433 ifeq ($(ICONFIG),TOKAMAK)
434  ICONFIG_OPT = -D_TOKAMAK
435  OPT += $(ICONFIG_OPT)
436 else ifeq ($(ICONFIG),CYLINDER)
437  ICONFIG_OPT = -D_CYLINDER
438  OPT += $(ICONFIG_OPT)
439 else ifeq ($(ICONFIG),TOROIDAL3D)
440  ICONFIG_OPT = -D_TOROIDAL3D
441  OPT += $(ICONFIG_OPT)
442 endif
443 
444 ifeq ($(SDP_OUT),y)
445  OPT += -D_SDP
446 endif
447 
448 # NetCDF library is always required
449 ifeq ($(NETCDF),y)
450  NETCDF_HOME=/exports/apps/installed/spack/opt/spack/linux-rocky8-zen/gcc-8.5.0/netcdf-fortran-4.6.0-65d4q7c6xulphalt2lpacg3ytactn7g3
451  NETCDF_LIB := -I$(NETCDF_HOME)/include
452  NETCDF_LD_LIB := -L$(NETCDF_HOME)/lib -lnetcdff
453  LIB += $(NETCDF_LIB)
454  LD_LIB += $(NETCDF_LD_LIB)
455  OPT += -D_NETCDF
456 endif
457 ifeq ($(ADIOS2), y)
458  OPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
459 # OPTIMOPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
460  LD_LIB += ${ADIOS2_LIBS}
461 endif
462 
463 ifeq ($(PETSc),y)
464  PETSC_solver ?= petsc.o
465  PETSC_OPT += -D_PETSc -D_USING_EXT_SOLVER -lpetsc
466  OPT += $(PETSC_OPT)
467  PETSC_HOME=/exports/apps/installed/spack/opt/spack/linux-rocky8-zen/gcc-8.5.0/petsc-3.18.1-xeuns4w5kms5jertyhfmlqwtmlmyhw3m
468  LIB += -I$(PETSC_HOME)/include
469  LD_LIB += $(PETSC_LD_LIB) -L$(PETSC_HOME)/lib
470  ifeq ($(AMGX),y)
471  $(error Currently cannot enable multiple solvers in a single GTC build)
472  endif
473  ifeq ($(HYPRE),y)
474  $(error Currently cannot enable multiple solvers in a single GTC build)
475  endif
476 else
477  PETSC_solver :=
478 endif
479 
480 ifeq ($(AMGX),y)
481  AMGX_solver ?= amgx.o
482  AMGX_OPT += -D_AMGX -D_USING_EXT_SOLVER
483  ifeq ($(DOUBLE_PRECISION),y)
484  AMG_OPT += -DDOUBLE_PRECISION
485  endif
486  OPT += $(AMGX_OPT)
487  LIB += $(AMGX_LIB)
488  LD_LIB += $(AMGX_LD_LIB)
489  ifeq ($(PETSc),y)
490  $(error Currently cannot enable multiple solvers in a single GTC build)
491  endif
492  ifeq ($(HYPRE),y)
493  $(error Currently cannot enable multiple solvers in a single GTC build)
494  endif
495 else
496  AMGX_solver :=
497 endif
498 
499 ifeq ($(HYPRE),y)
500  HYPRE_solver ?= hypre.o
501  HYPRE_OPT += -D_HYPRE -D_USING_EXT_SOLVER
502  ifeq ($(DOUBLE_PRECISION),y)
503  HYPRE_OPT += -DDOUBLE_PRECISION
504  endif
505  ifeq ($(HYPRE_USES_GPU),y)
506  HYPRE_OPT += -D_HYPRE_USES_GPU
507  endif
508  OPT += $(HYPRE_OPT)
509  COPT += $(HYPRE_OPT)
510  LIB += $(HYPRE_LIB)
511  LD_LIB += $(HYPRE_LD_LIB)
512  ifeq ($(PETSc),y)
513  $(error Currently cannot enable multiple solvers in a single GTC build)
514  endif
515  ifeq ($(AMGX),y)
516  $(error Currently cannot enable multiple solvers in a single GTC build)
517  endif
518 else
519  HYPRE_solver :=
520 endif
521 
522 OMPOPT := -fopenmp
523 ifeq ($(OPENMP),y)
524  OPT += $(OMPOPT)
525  COPT += $(OMPOPT)
526 endif
527 
528 ifeq ($(DOUBLE_PRECISION),y)
529  OPT += -DDOUBLE_PRECISION $(DPOPT)
530 endif
531 
532 ifeq ($(DEBUG),y)
533  OPT += $(DEBUGOPT)
534  COPT += $(DEBUGCOPT)
535 else
536  OPT += $(OPTIMOPT)
537 # COPT += $(OPTIMOPT)
538 endif
539 
540 ifeq ($(GPU),y)
541  OPT:= -Minfo=accel -acc=verystrict $(OPT)
542  ifneq (,$(findstring perlmutter,$(SYSTEMS)))
543  CUDA_OPT += -O3
544  ifeq ($(CUDA_MPI),y)
545  OPT += -target-accel=nvidia80
546  CUDA_OPT += -target-accel=nvidia80
547  endif
548  LD_LIB += -L$(NVIDIA_PATH)/cuda/lib64 -L$(NVIDIA_PATH)/math_libs/lib64/
549  endif
550  ifneq (,$(findstring ppc64le,$(SYSTEMS)))
551  ifeq ($(GPU_UM),y)
552  OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -DGPU_UM -Mcuda=9.0 -ta=nvidia:cc70,maxregcount:160,managed $(OPT)
553  CUDA_OPT += -O3 -arch=sm_70
554  else
555  OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -ta=nvidia:cc70,maxregcount:160 $(OPT)
556  CUDA_OPT += -O3 -arch=sm_70
557  endif
558  endif
559  ifeq ($(DOUBLE_PRECISION),y)
560  CUDA_OPT += -DDOUBLE_PRECISION
561  endif
562 
563  CUDA_OBJ := shift_cuda.o cufft.o
564  LD_LIB += $(CUDA_LIB)
565  ifeq ($(CUDA_MPI),y)
566  OPT += -DCUDA_MPI
567  CUDA_OPT += -DCUDA_MPI
568  endif
569  ifeq ($(CUDA_MPI),y)
570  OPT += -DCUDA_MPI
571  CUDA_OPT += -DCUDA_MPI
572  endif
573 else
574  OPT := $(OPT)
575 endif
576 
577 ifeq ($(TIME_MPI),y)
578  OPT := -DTIME_MPI $(OPT)
579  CUDA_OPT += -DTIME_MPI
580 endif
581 
582 # Equilibrium dimension related OBJs
583 ifeq ($(ICONFIG), TOROIDAL3D)
584  EQ_OBJ := VMEC_interface.o
585 else
586 ifeq ($(NETCDF),y)
587  EQ_OBJ := M3DC1_interface.o
588 endif
589 endif
590 
591 CMP := mpifort
592 ##################################################################
593 # We add ".F90" to the list of suffixes to allow source files on which the
594 # co-processor will be run automatically.
595 .SUFFIXES: .o .F90 .F
596 .PHONY: clean
597 
598 # List of all the object files needed to build the code
599 OBJ:=simple_error_handler.o module.o function.o boundary_decay.o main.o setup.o restart.o\
600  diagnosis.o snapshot.o phishot.o poisson.o smooth.o field.o field_operators.o\
601  bparasolver.o tracking.o dataout3d.o pushfield.o eqdata.o eqplot.o collision.o\
602  linsolver.o $(PETSC_solver) $(AMGX_solver) $(HYPRE_solver) fft_gl.o load.o push.o\
603  shift.o charge.o pressure_push.o radial_bin.o $(CUDA_OBJ) antenna.o utility.o ohm_law.o\
604  physical_meaning.o $(EQ_OBJ)
605 
606 ifeq ($(ADIOS2), y)
607  OBJ += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o m_dataout3d_adios.o
608 endif
609 
610 HYBRID_OBJ:=hybridChargeParticle.o hybridPushParticle.o
611 KINETIC_OBJ:=gkChargeParticle.o gkPushParticle.o
612 FKI_OBJ:=fkChargeParticle.o PushFKI.o
613 
614 OBJ += $(HYBRID_OBJ) $(KINETIC_OBJ) $(FKI_OBJ)
615 
616 ifeq ($(SDP_OUT),y)
617  OBJ += SDP_interface.o
618 endif
619 
620 # set dependencies on modules based on switches
621 DEPEND_MOD :=
622 ifeq ($(SDP_OUT),y)
623  DEPEND_MOD += SDP_interface.o
624 endif
625 
626 # set commonly depended modules
627 COMMON_MOD := module.o utility.o simple_error_handler.o
628 ifeq ($(ADIOS2), y)
629  COMMON_MOD += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o m_dataout3d_adios.o
630 endif
631 
632 # Start building the executive
633 $(CMD): $(OBJ)
634  $(CMP) $(OPT) -o $(CMD) $(OBJ) $(LD_LIB)
635 
636 $(filter-out module.o,$(OBJ)): module.o
637 
638 simple_error_handler.o : simple_error_handler.F90
639  $(CMP) $(OPT) $(LIB) -c simple_error_handler.F90
640 
641 module.o : module.F90
642  $(CMP) $(OPT) $(LIB) -c module.F90
643 
644 utility.o : utility.F90 module.o
645  $(CMP) $(OPT) $(LIB) -c utility.F90
646 
647 adios2_common.o : module.o simple_error_handler.o adios2_common.F90
648  $(CMP) $(OPT) $(LIB) -c adios2_common.F90
649 
650 adios2_history_diagnostics.o : adios2_common.o module.o adios2_history_diagnostics.F90
651  $(CMP) $(OPT) $(LIB) -c adios2_history_diagnostics.F90
652 
653 adios2_data1d_diagnostics.o : adios2_common.o module.o adios2_data1d_diagnostics.F90
654  $(CMP) $(OPT) $(LIB) -c adios2_data1d_diagnostics.F90
655 
656 m_dataout3d_adios.o : adios2_common.o module.o simple_error_handler.o m_dataout3d_adios.F90
657  $(CMP) $(OPT) $(LIB) -c m_dataout3d_adios.F90
658 
659 main.o : main.F90 $(COMMON_MOD) diagnosis.o antenna.o
660  $(CMP) $(OPT) $(LIB) -c main.F90
661 
662 function.o: function.F90 $(COMMON_MOD)
663  $(CMP) $(OPT) $(LIB) -c function.F90
664 
665 boundary_decay.o : boundary_decay.F90 $(COMMON_MOD) function.o
666  $(CMP) $(OPT) $(LIB) -c boundary_decay.F90
667 
668 antenna.o: antenna.F90 $(COMMON_MOD)
669  $(CMP) $(OPT) $(LIB) -c antenna.F90
670 
671 ifeq ($(NETCDF),y)
672 M3DC1_interface.o: M3DC1_interface.F90 $(COMMON_MOD) function.o boundary_decay.o
673  $(CMP) $(OPT) $(LIB) -c M3DC1_interface.F90
674 endif
675 
676 VMEC_interface.o: VMEC_interface.F90 $(COMMON_MOD) function.o
677  $(CMP) $(OPT) $(LIB) -c VMEC_interface.F90
678 
679 eqdata.o: eqdata.F90 $(EQ_OBJ) $(COMMON_MOD) function.o boundary_decay.o $(DEPEND_MOD)
680  $(CMP) $(OPT) $(LIB) -c eqdata.F90
681 
682 eqplot.o: eqplot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD)
683  $(CMP) $(OPT) $(LIB) -c eqplot.F90
684 
685 snapshot.o: snapshot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD) physical_meaning.o
686  $(CMP) $(OPT) $(LIB) -c snapshot.F90
687 
688 phishot.o: phishot.F90 $(COMMON_MOD)
689  $(CMP) $(OPT) $(LIB) -c phishot.F90
690 
691 dataout3d.o: dataout3d.F90 $(COMMON_MOD) function.o
692  $(CMP) $(OPT) $(LIB) -c dataout3d.F90
693 
694 SDP_interface.o: SDP_interface.F90 $(COMMON_MOD) function.o
695  $(CMP) $(OPT) $(LIB) -c SDP_interface.F90
696 
697 linsolver.o: linsolver.F90 $(COMMON_MOD) poisson.o ohm_law.o bparasolver.o
698  $(CMP) $(OPT) $(LIB) -c linsolver.F90
699 
700 poisson.o: poisson.F90 $(COMMON_MOD) function.o antenna.o
701  $(CMP) $(OPT) $(LIB) -c poisson.F90
702 
703 pushfield.o: pushfield.F90 $(COMMON_MOD) function.o antenna.o
704  $(CMP) $(OPT) $(LIB) -c pushfield.F90
705 
706 ohm_law.o: ohm_law.F90 $(COMMON_MOD) function.o
707  $(CMP) $(OPT) $(LIB) -c ohm_law.F90
708 
709 field.o: field.F90 $(COMMON_MOD) antenna.o
710  $(CMP) $(OPT) $(LIB) -c field.F90
711 
712 field_operators.o: field_operators.F90 $(COMMON_MOD) antenna.o
713  $(CMP) $(OPT) $(LIB) -c field_operators.F90
714 
715 setup.o: setup.F90 $(COMMON_MOD) function.o boundary_decay.o antenna.o $(EQ_OBJ) diagnosis.o physical_meaning.o
716  $(CMP) $(OPT) $(LIB) -c setup.F90
717 
718 smooth.o: smooth.F90 $(COMMON_MOD) boundary_decay.o
719  $(CMP) $(OPT) $(LIB) -c smooth.F90
720 
721 load.o: load.F90 $(COMMON_MOD) function.o antenna.o
722  $(CMP) $(OPT) $(LIB) -c load.F90
723 
724 tracking.o: tracking.F90 $(COMMON_MOD) function.o
725  $(CMP) $(OPT) $(LIB) -c tracking.F90
726 
727 #json_module.o: json_module.F90
728 # $(CMP) $(OPT) $(LIB) -c json_module.F90
729 
730 hybrid%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
731  $(FCMP) $(OPT) $(LIB) -D_hybrid $(PREPROCESS_OPT) $*.F90 > $@
732  mv $*.i $@ || true
733 
734 gk%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
735  $(FCMP) $(OPT) $(LIB) $(PREPROCESS_OPT) $*.F90 > $@
736  mv $*.i $@ || true
737 
738 fk%.F90 : $(COMMON_MOD) boundary_decay.o function.o ChargeParticle.F90
739  $(FCMP) $(OPT) $(LIB) -D_fk $(PREPROCESS_OPT) $*.F90 > $@
740  mv $*.i $@ || true
741 
742 PushFKI.o: PushFKI.F90 $(COMMON_MOD) boundary_decay.o function.o
743  $(CMP) $(OPT) $(LIB) -c PushFKI.F90
744 
745 shift_cuda.o : shift_cuda.cu shift_cuda.cuh
746  $(NVCC) $(CUDA_OPT) -c shift_cuda.cu
747 
748 cufft.o : cufft.cu
749  $(NVCC) $(CUDA_OPT) -c cufft.cu
750 
751 amgx.o: amgx.cu
752  $(NVCC) $(CUDA_OPT) -c amgx.cu $(AMGX_LIB) $(AMGX_OPT)
753 
754 hypre.o: hypre.c
755  $(CCMP) $(COPT) -c hypre.c $(HYPRE_LIB)
756 
757 diagnosis.o: diagnosis.F90 $(COMMON_MOD) function.o antenna.o
758  $(CMP) $(OPT) $(LIB) -c diagnosis.F90
759 
760 shift.o: shift.F90 $(COMMON_MOD)
761  $(CMP) $(OPT) $(LIB) -c shift.F90
762 
763 restart.o: restart.F90 $(COMMON_MOD)
764  $(CMP) $(OPT) $(LIB) -c restart.F90
765 
766 physical_meaning.o : $(COMMON_MOD)
767  $(CMP) $(OPT) $(LIB) -c physical_meaning.F90
768 
769 .F90.o : $(COMMON_MOD)
770  $(CMP) $(OPT) $(LIB) -c $<
771 
772 # The following tag is meant to "clean" the directory by removing the
773 # executable along with all the object files created by the compilation
774 # One only has to run: gmake clean
775 
776 clean::
777  rm -f $(CMD) $(OBJ) *.mod *.o *genmod.f90 *.optrpt *.json pgipar* *.dbg *.smod
778 
779 print-%: ; @echo $* = $($*)
780 
781 tags:
782  find . -name "*.F90" -o -name "*.c" -o -name "*.cu" -o -name "*.h??" | etags -
783