Essentials of running GTC
Bashrc | Job-Script | Makefile - To run GTC(v4.6U) in ParamPravega and PTG
Makefile_Param
Go to the documentation of this file.
1 # This file is part of GTC version 4.6
2 # GTC version 4.6 is released under the 3-Clause BSD license:
3 
4 # Copyright (c) 2002,2010,2016, GTC Team (team leader: Zhihong Lin, zhihongl@uci.edu)
5 # All rights reserved.
6 
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are met:
9 
10 # 1. Redistributions of source code must retain the above copyright notice,
11 # this list of conditions and the following disclaimer.
12 
13 # 2. Redistributions in binary form must reproduce the above copyright notice,
14 # this list of conditions and the following disclaimer in the documentation
15 # and/or other materials provided with the distribution.
16 
17 # 3. Neither the name of the GTC Team nor the names of its contributors may be
18 # used to endorse or promote products derived from this software without
19 # specific prior written permission.
20 # ==============================================================================
21 
22 #key parameters:
23 OPENMP=y
24 DOUBLE_PRECISION=n
25 ADIOS2=n
26 DEBUG=n
27 # set SDP_OUT to y to enable synthetic diagnostic platform outputs
28 SDP_OUT=n
29 COMPILER=default
30 #ICONFIG=TOKAMAK
31 #ICONFIG=CYLINDER
32 ICONFIG=TOROIDAL3D
33 
34 # GPU and field solver flags
35 NETCDF=y
36 GPU=n
37 GPU_UM=n
38 PETSc=y
39 AMGX=n
40 HYPRE=n
41 HYPRE_USES_GPU=n
42 TIME_MPI=y
43 NVTX=n
44 CUDA_MPI=n
45 ############################################################################
46 # Makefile to build the GTC code
47 # ==================================
48 #
49 # You only need to type "gmake" to build the code on the platforms
50 # defined below. The makefile runs the "uname -s" command to detect
51 # the operating system automatically. Other options are:
52 #
53 # % gmake OPENMP=y Builds the code with OpenMP support
54 # % gmake OPENMP=n Builds the code WITHOUT OpenMP support
55 # The default is with OpenMP support
56 #
57 # % gmake DOUBLE_PRECISION=y Builds with 8-byte floating point precision
58 # The default is single precision
59 #
60 # % gmake PETSc=y Builds the code using the PETSc parallel matrix
61 # solving library. This is necessary to run the
62 # electromagnetic version of GTC.
63 #
64 # % gmake DEBUG=y Compiles the files with debugging flags.
65 # The default is no debug option.
66 #
67 # If COMPILER is set to 'default' the Makefile will select the appropriate
68 # compiler for the target. If the user wished to override the target
69 # machine default, COMPILER may be set to 'portland', 'intel', or 'gnu'.
70 #
71 # Special targets:
72 #
73 # % gmake clean Removes the executable and all object files (*.o)
74 #
75 # % gmake cleanomp Removes the executable and the object files
76 # containing OpenMP directives
77 #
78 # % gmake doc Rebuilds the documentation.
79 #
80 #############################################################################
81 # Default executable name
82 CMD := gtc
83 ALL:${CMD}
84 LIB :=
85 LD_LIB :=
86 
87 ##### inquire the hostname to judge the system name, to determine what target we're compiling on
88 ifneq (,${NERSC_HOST})
89  SYSTEMS := ${NERSC_HOST}
90 else ifneq (,${HOST})
91  SYSTEMS := ${HOST}
92 else
93  ifneq (,$(findstring ppc64le,$(shell uname -m)))
94  SYSTEMS := ppc64le
95  else
96  SYSTEMS := $(shell hostname)
97  endif
98 endif
99 
100 ##### target machine determines the default compiler, and paths to libraries #####
101 # theta
102 ifneq (,$(findstring thetalogin,$(SYSTEMS)))
103  SYSTEM_FOUND=y
104  ifeq ($(COMPILER),default)
105  COMPILER := cray
106  endif
107  CMP := ftn
108  CCMP := cc
109 # module swap PrgEnv-intel PrgEnv-cray
110 # module load cray-petsc/3.12.4.1 cray-netcdf-hdf5parallel/4.7.3.3
111 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
112  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
113 # PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
114  NETCDF_LIB = -I${NETCDF_DIR}/include
115  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
116  ifeq ($(HYPRE),y)
117  HYPRE_LIB = -I${HYPRE_ROOT}/src/hypre/include
118  HYPRE_LD_LIB = -L${HYPRE_ROOT}/src/hypre/lib -lHYPRE
119  endif
120 endif
121 
122 # thetagpu
123 ifneq (,$(findstring thetagpu,$(SYSTEMS)))
124  SYSTEM_FOUND=y
125  ifeq ($(COMPILER),default)
126  COMPILER := portland
127  endif
128  CMP := mpif90
129  CCMP := mpicc
130  NVCC := nvcc
131 # module swap PrgEnv-intel PrgEnv-cray
132 # module load cray-petsc/3.12.4.1 cray-netcdf-hdf5parallel/4.7.3.3
133 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
134  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
135 # PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
136  NETCDF_LIB = -I${NETCDF_DIR}/include
137  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf ${NETCDF_DIR}/lib/libnetcdff.a
138  CUDA_LIB := -lstdc++ -L/soft/hpc-sdk/Linux_x86_64/21.7/math_libs/lib64 -L/soft/hpc-sdk/Linux_x86_64/21.7/cuda/lib64 -lcudart -lcufft -lnvToolsExt
139  ifeq ($(HYPRE),y)
140  HYPRE_LIB = -I${HYPRE_ROOT}/src/hypre/include
141  HYPRE_LD_LIB = -L${HYPRE_ROOT}/src/hypre/lib -lHYPRE
142  endif
143 endif
144 # cori
145 ifneq (,$(findstring cori,$(SYSTEMS)))
146  ifeq ($(COMPILER),default)
147  COMPILER := intel
148  endif
149  CMP := ftn
150  CCMP := cc
151 # PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
152  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
153  ifeq ($(NETCDF),y)
154  NETCDF_LIB = -I${NETCDF_DIR}/include
155  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
156  endif
157  LIBD = /global/u2/x/xiao/AdioFranklin/lib
158  INCD = /global/u2/x/xiao/AdioFranklin/include
159  ADIOS2_DIR = /project/projectdirs/m808/kmehta/adios2-12.16.18/installation/intel/18.0.1
160  ADIOS2_LIBS = -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet -ladios2_atl -lstdc++
161  ifeq ($(HYPRE),y)
162  HYPRE_LD_LIB = -L${HYPRE_ROOT}/lib -lHYPRE
163  HYPRE_LIB = -I${HYPRE_INC}
164  endif
165 endif
166 
167 #perlmutter
168 ifneq (,$(findstring perlmutter,$(SYSTEMS)))
169  ifeq ($(COMPILER),default)
170  COMPILER := portland
171  DEBUG_OPT_ := -g -C -gopt -Mchkstk -Mcoff -Mdwarf3 -Melf -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
172  endif
173 
174  ifeq ($(GPU),y)
175  COMPILER := portland
176  NVCC := CC -cuda
177  CUDA_LIB := -lstdc++ -lcudart -lcufft -cuda -rpath=/opt/cray/libfabric/1.11.0.4.75/lib64
178  DEBUG_OPT_ := -g -C -gopt -Mchkstk -Mcoff -Mdwarf3 -Melf -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
179  endif
180  CMP := ftn
181  CCMP := cc
182  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
183  ifeq ($(NETCDF),y)
184  NETCDF_LIB = -I${NETCDF_DIR}/include
185  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
186  endif
187  LIBD = /global/u2/x/xiao/AdioFranklin/lib
188  INCD = /global/u2/x/xiao/AdioFranklin/include
189  ADIOS2_DIR = /project/projectdirs/m808/kmehta/adios2-12.16.18/installation/intel/18.0.1
190  ADIOS2_LIBS = -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet -ladios2_atl -lstdc++
191  ifeq ($(HYPRE),y)
192  HYPRE_LIB = -I${HYPRE_INC}
193  HYPRE_LD_LIB = ${HYPRE_LFLAGS}
194  endif
195 endif
196 
197 # summit
198 #ifneq (,$(findstring summit,$(SYSTEMS))) # can not work now
199 #ifneq (,$(findstring ppc64le,$(shell uname -m)))
200 ifneq (,$(findstring ppc64le,$(SYSTEMS)))
201  ifeq ($(OLCF_AMGX_ROOT),)
202  AMGX_ROOT = /ccs/proj/csc190/gtc/amgx/summit
203  else
204  AMGX_ROOT = $(OLCF_AMGX_ROOT)
205  endif
206  ifeq ($(OLCF_HYPRE_ROOT),)
207  ifeq ($(HYPRE_USES_GPU),y)
208  HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre_gpu/summit
209  else
210  HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre/summit
211  endif
212  else
213  #ifeq ($(HYPRE_USES_GPU),y)
214  # $(error To use GPU-enabled Hypre, please unload the hypre module and use the custom hypre build)
215  #endif
216  HYPRE_ROOT = $(OLCF_HYPRE_ROOT)
217  endif
218  CMP := mpif90
219  CCMP := mpicc
220  FCMP := gfortran
221  ifeq ($(COMPILER),default)
222  COMPILER := portland
223  ifeq ($(GPU), y)
224  CUDA_LIB := -lstdc++ -L$(dir $(shell which nvcc))../lib64 -lcudart -lcufft -lnvToolsExt
225  NVCC := nvcc
226  endif
227  DEBUG_OPT_ := -g -C -gopt -Mchkfpstk -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
228  FCMP := pgf90
229  endif
230  ifeq ($(PETSc),y)
231  PETSC_LIB = -I${OLCF_PETSC_ROOT}/include
232  PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
233  endif
234  ifeq ($(NETCDF),y)
235  NETCDF_LIB = -I${OLCF_NETCDF_FORTRAN_ROOT}/include
236  NETCDF_LD_LIB = -L${OLCF_NETCDF_FORTRAN_ROOT}/lib -R${OLCF_NETCDF_FORTRAN_ROOT}/lib -lnetcdff -L${OLCF_CUDA_ROOT}/lib64/ -lnvToolsExt
237  endif
238  HDF5_LIB = -I${OLCF_HDF5_ROOT}/include
239  HDF5_LD_LIB = -L${OLCF_HDF5_ROOT}/lib -R${OLCF_HDF5_ROOT}/lib -lhdf5_fortran
240  MPI_LIB := -I${OLCF_OMPI_ROOT}/include
241  CUDA_OPT := -I$(dir $(shell which mpif90))../include
242  ifeq ($(AMGX),y)
243  AMGX_LIB = -I${AMGX_ROOT}/../AMGX/base/include
244  AMGX_LD_LIB = -L${AMGX_ROOT}/lib -R${AMGX_ROOT}/lib -lamgxsh -L$(CUDA_DIR)/targets/ppc64le-linux/lib -R$(CUDA_DIR)/targets/ppc64le-linux/lib -lcusparse
245  endif
246  ifeq ($(HYPRE),y)
247  HYPRE_LIB = -I${HYPRE_ROOT}/hypre/src/hypre//include -I${HYPRE_ROOT}/include
248  HYPRE_LD_LIB = -L${HYPRE_ROOT}/hypre/src/hypre/lib -R${HYPRE_ROOT}/hypre/src/hypre/lib -L${HYPRE_ROOT}/lib -R${HYPRE_ROOT}/lib -lHYPRE
249  ifeq ($(HYPRE_USES_GPU),y)
250  HYPRE_LD_LIB += -L${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -R${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -lnvToolsExt
251  endif
252  endif
253 # ADIOS2_DIR=/ccs/proj/phy130/wenhao/adios2-2.6.0/installation
254  ADIOS2_DIR=/gpfs/alpine/world-shared/csc143/kmehta/ADIOS2-2.8.3/installation/pgi/20.4
255  #ADIOS2_DIR=/gpfs/alpine/world-shared/csc143/kmehta/ADIOS2-2.5.0/installation/pgi/19.9/
256  #ADIOS2_DIR=${OLCF_ADIOS2_ROOT}
257  ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)
258 endif
259 
260 # power
261 ifneq (,$(findstring power,$(SYSTEMS)))
262  COMPILER := portland
263  CMP :=mpif90
264 # PETSC_OPT = -D_PETSc35BEFORE##only needed for kink/tearing
265  PETSC_LIB = -I${PETSC_DIR}/include
266  PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
267  CUDA_OPT := -I$(dir $(shell which mpif90))../include
268 # NETCDF_LIB = -I${NETCDF_DIR}/include
269 # NETCDF_LD_LIB = -L${NETCDF_DIR}/lib
270 # LIBD = /ccs/home/hardes/gtc_adios/jaguarlib
271 # INCD = /ccs/home/hardes/gtc_adios/jaguarinclude
272 # ADIOS_LIB = -I$(INCD)
273 # ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
274 endif
275 
276 # Tianhe-1A
277 ifneq (,$(findstring ln,$(SYSTEMS)))
278  ifeq ($(COMPILER),default)
279  ifeq ($(GPU),y)
280  COMPILER := portland
281  else
282  COMPILER := intel
283  endif
284  endif
285  CMP := mpif90
286 
287  PETSC_solver = petsc.o
288  PETSC_OPT = -D_PETSc31P8ANDBEFORE
289  #PETSC_OPT += -D_PETSc30ANDBEFORE
290  ifneq (,${PETSC_DIR})
291  include ${PETSC_DIR}/conf/variables
292  endif
293 
294  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
295  PETSC_LD_LIB := ${PETSC_KSP_LIB}
296 
297  NETCDF_LIB := -I${NETCDF_DIR}/include
298  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
299  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
300 endif
301 
302 # PKU Weiming, cluster
303 ifneq (,$(findstring pku,$(SYSTEMS)))
304  ifeq ($(COMPILER),default)
305  COMPILER := intel
306  endif
307  CMP := mpiifort
308 
309  #PETSC_solver = petsc.o
310  #PETSC_OPT = -D_PETSc31P8ANDBEFORE
311  #PETSC_OPT += -D_PETSc30ANDBEFORE
312  ifneq (,${PETSC_DIR})
313  # include ${PETSC_DIR}/conf/variables
314  include ${PETSC_DIR}/lib/petsc/conf/variables
315  endif
316 
317  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
318  PETSC_LD_LIB := ${PETSC_KSP_LIB}
319 
320  NETCDF_LIB := -I${NETCDF_DIR}/include
321  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff \
322  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
323 endif
324 
325 #sun
326 ifneq (,$(findstring sun,$(SYSTEMS)))
327  ifeq ($(COMPILER),default)
328  COMPILER := gnu
329  endif
330  CMP := mpif90
331  CCMP := mpicc
332  HDF5_INCLUDE := -I/opt/hdf5_parallel/include
333 # PETSC_INCLUDE := -I${PETSC_DIR}/include -I${PETSC_DIR}/${PETSC_ARCH}/include -I${PETSC_DIR}/include/petsc/finclude
334  HDF5_LIB := -lhdf5_fortran -L/opt/hdf5_parallel/lib
335  PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/petsc/finclude -I${PETSC_DIR}/${PETSC_ARCH}/include
336  PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
337 
338 # ADIOS2_DIR=/home/kmehta/ADIOS2-2.6.0/installation/
339 # ADIOS2_DIR=/home/wenhao/Downloads/adios2-2.6.0/installation
340 # ADIOS2_DIR=/opt/adios-2.6.0/installation
341  ADIOS2_DIR=/store/home/wenhao/Downloads/adios2-2.8.3/installation
342  ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)
343  ifeq ($(HYPRE),y)
344  HYPRE_LD_LIB = -L${HYPRE_LD} -lHYPRE
345  HYPRE_LIB = -I${HYPRE_INC}
346  endif
347  ifeq ($(NETCDF),y)
348  NETCDF_DIR = /usr
349  NETCDF_LIB = -I${NETCDF_DIR}/include/
350  NETCDF_LD_LIB = -L${NETCDF_DIR}/lib/ -lnetcdff
351  endif
352 endif
353 
354 #iop
355 ifneq (,$(findstring indac,$(SYSTEMS)))
356  ifeq ($(COMPILER),default)
357  COMPILER := intel
358  endif
359  CMP := mpif90
360 
361  #PETSC_solver = petsc.o
362  PETSC_OPT = -D_PETSc31P8ANDBEFORE
363  #PETSC_OPT += -D_PETSc30ANDBEFORE
364  ifneq (,${PETSC_DIR})
365  include ${PETSC_DIR}/conf/variables
366  endif
367 
368  PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
369  PETSC_LD_LIB := ${PETSC_KSP_LIB}
370  ifeq ($(NETCDF),y)
371  NETCDF_LIB := -I${NETCDF_DIR}/include
372  NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
373  -Wl,--rpath -Wl,${NETCDF_DIR}/lib
374  endif
375 endif
376 
377 #### compiler determines debugging, openmp, and double precision flags #####
378 # intel fortran
379 ifeq ($(COMPILER),intel)
380  OPTIMOPT := -O
381  #OPTIMOPT := -O3
382  #OPTIMOPT += -fp-model fast=2
383  #OPTIMOPT += -align array64byte
384  #OPTIMOPT += -no-vec
385 # debug flags changed by L. Shi to suppress long warning outputs.
386  #DEBUGOPT := -g -free -pc64 -check all -warn nounused -warn all -debug all -debug-parameters all -fp-stack-check
387  DEBUGOPT := -g -free -pc64 -check 'all,noarg_temp_created' -warn none -debug all -debug-parameters all -fp-stack-check
388  DEBUGCOPT := -O0 -g
389  OMPOPT := -qopenmp
390  DPOPT := -fpconstant
391 endif
392 
393 # portland group fortran
394 ifeq ($(COMPILER),portland)
395  OPTIMOPT := -fast -Mfree -Kieee
396  #DEBUGOPT := -g -C -gopt -Mbounds -Mchkfpstk -Mchkptr -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -traceback -Minform=inform
397  # -Mchkptr is currently not working well with GPU compiling, so temporarily opted out
398  DEBUGOPT := $(DEBUG_OPT_)
399  DEBUGCOPT := -O0 -g
400  OMPOPT := -mp
401  DPOPT := -DDOUBLE_PRECISION
402 endif
403 
404 ### assemble all the options
405 OPT :=
406 LIB :=
407 PREPROCESS_OPT := -E
408 FCMP ?= $(CMP)
409 
410 # GNU gfortran
411 ifeq ($(COMPILER),gnu)
412  OPT += -ffree-line-length-0 -Wuninitialized
413  OPTIMOPT := -O
414  DEBUGOPT := -g -fcheck=all,no-array-temps,no-bounds
415  DEBUGCOPT := -O0 -g -Wall
416 # DEBUGOPT := -g -fcheck=all,no-array-temps
417  OMPOPT := -fopenmp
418  DPOPT :=
419 endif
420 
421 # ALCF Theta Cray ftn (KNL, no OpenACC)
422 ifeq ($(COMPILER),cray)
423  # no -N0 option comparable to -ffree-line-length-0
424  OPT += -xacc -h noacc -N1023 -O shortcircuit2 -Werror
425  OPTIMOPT := -O1
426  DEBUGOPT := -g -nobounds
427  DEBUGCOPT := -O0 -g -Wall
428  OMPOPT := -homp
429  DPOPT :=
430  PREPROCESS_OPT := -eP
431 endif
432 
433 ifeq ($(ICONFIG),TOKAMAK)
434  ICONFIG_OPT = -D_TOKAMAK
435  OPT += $(ICONFIG_OPT)
436 else ifeq ($(ICONFIG),CYLINDER)
437  ICONFIG_OPT = -D_CYLINDER
438  OPT += $(ICONFIG_OPT)
439 else ifeq ($(ICONFIG),TOROIDAL3D)
440  ICONFIG_OPT = -D_TOROIDAL3D
441  OPT += $(ICONFIG_OPT)
442 endif
443 
444 ifeq ($(SDP_OUT),y)
445  OPT += -D_SDP
446 endif
447 
448 # NetCDF library is always required
449 ifeq ($(NETCDF),y)
450 ## NETCDF_HOME=/exports/apps/installed/spack/opt/spack/linux-rocky8-zen/gcc-8.5.0/netcdf-fortran-4.6.0-65d4q7c6xulphalt2lpacg3ytactn7g3 PTG
451  NETCDF_HOME=/home/ext/apps/spack/opt/spack/linux-centos7-skylake_avx512/intel-18.0.5/netcdf-fortran-4.5.3-mxkpjlifwez6uij2qnz2pmn5kavlmpng
452  NETCDF_LIB := -I$(NETCDF_HOME)/include
453  NETCDF_LD_LIB := -L$(NETCDF_HOME)/lib -lnetcdff
454  LIB += $(NETCDF_LIB)
455  LD_LIB += $(NETCDF_LD_LIB)
456  OPT += -D_NETCDF
457 endif
458 ifeq ($(ADIOS2), y)
459  OPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
460 # OPTIMOPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
461  LD_LIB += ${ADIOS2_LIBS}
462 endif
463 
464 ifeq ($(PETSc),y)
465  PETSC_solver ?= petsc.o
466  PETSC_OPT += -D_PETSc -D_USING_EXT_SOLVER -lpetsc
467  OPT += $(PETSC_OPT)
468 ## PETSC_HOME=/exports/apps/installed/spack/opt/spack/linux-rocky8-zen/gcc-8.5.0/petsc-3.18.1-xeuns4w5kms5jertyhfmlqwtmlmyhw3m PTG
469  PETSC_HOME=/home/ext/apps/spack/opt/spack/linux-centos7-cascadelake/intel-2021.4.0/petsc-3.16.2-qb3bzznpghhokvmeitoebasgnvct5sau
470  LIB += -I$(PETSC_HOME)/include
471  LD_LIB += $(PETSC_LD_LIB) -L$(PETSC_HOME)/lib
472  ifeq ($(AMGX),y)
473  $(error Currently cannot enable multiple solvers in a single GTC build)
474  endif
475  ifeq ($(HYPRE),y)
476  $(error Currently cannot enable multiple solvers in a single GTC build)
477  endif
478 else
479  PETSC_solver :=
480 endif
481 
482 ifeq ($(AMGX),y)
483  AMGX_solver ?= amgx.o
484  AMGX_OPT += -D_AMGX -D_USING_EXT_SOLVER
485  ifeq ($(DOUBLE_PRECISION),y)
486  AMG_OPT += -DDOUBLE_PRECISION
487  endif
488  OPT += $(AMGX_OPT)
489  LIB += $(AMGX_LIB)
490  LD_LIB += $(AMGX_LD_LIB)
491  ifeq ($(PETSc),y)
492  $(error Currently cannot enable multiple solvers in a single GTC build)
493  endif
494  ifeq ($(HYPRE),y)
495  $(error Currently cannot enable multiple solvers in a single GTC build)
496  endif
497 else
498  AMGX_solver :=
499 endif
500 
501 ifeq ($(HYPRE),y)
502  HYPRE_solver ?= hypre.o
503  HYPRE_OPT += -D_HYPRE -D_USING_EXT_SOLVER
504  ifeq ($(DOUBLE_PRECISION),y)
505  HYPRE_OPT += -DDOUBLE_PRECISION
506  endif
507  ifeq ($(HYPRE_USES_GPU),y)
508  HYPRE_OPT += -D_HYPRE_USES_GPU
509  endif
510  OPT += $(HYPRE_OPT)
511  COPT += $(HYPRE_OPT)
512  LIB += $(HYPRE_LIB)
513  LD_LIB += $(HYPRE_LD_LIB)
514  ifeq ($(PETSc),y)
515  $(error Currently cannot enable multiple solvers in a single GTC build)
516  endif
517  ifeq ($(AMGX),y)
518  $(error Currently cannot enable multiple solvers in a single GTC build)
519  endif
520 else
521  HYPRE_solver :=
522 endif
523 
524 OMPOPT := -fopenmp
525 ifeq ($(OPENMP),y)
526  OPT += $(OMPOPT)
527  COPT += $(OMPOPT)
528 endif
529 
530 ifeq ($(DOUBLE_PRECISION),y)
531  OPT += -DDOUBLE_PRECISION $(DPOPT)
532 endif
533 
534 ifeq ($(DEBUG),y)
535  OPT += $(DEBUGOPT)
536  COPT += $(DEBUGCOPT)
537 else
538  OPT += $(OPTIMOPT)
539 # COPT += $(OPTIMOPT)
540 endif
541 
542 ifeq ($(GPU),y)
543  OPT:= -Minfo=accel -acc=verystrict $(OPT)
544  ifneq (,$(findstring perlmutter,$(SYSTEMS)))
545  CUDA_OPT += -O3
546  ifeq ($(CUDA_MPI),y)
547  OPT += -target-accel=nvidia80
548  CUDA_OPT += -target-accel=nvidia80
549  endif
550  LD_LIB += -L$(NVIDIA_PATH)/cuda/lib64 -L$(NVIDIA_PATH)/math_libs/lib64/
551  endif
552  ifneq (,$(findstring ppc64le,$(SYSTEMS)))
553  ifeq ($(GPU_UM),y)
554  OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -DGPU_UM -Mcuda=9.0 -ta=nvidia:cc70,maxregcount:160,managed $(OPT)
555  CUDA_OPT += -O3 -arch=sm_70
556  else
557  OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -ta=nvidia:cc70,maxregcount:160 $(OPT)
558  CUDA_OPT += -O3 -arch=sm_70
559  endif
560  endif
561  ifeq ($(DOUBLE_PRECISION),y)
562  CUDA_OPT += -DDOUBLE_PRECISION
563  endif
564 
565  CUDA_OBJ := shift_cuda.o cufft.o
566  LD_LIB += $(CUDA_LIB)
567  ifeq ($(CUDA_MPI),y)
568  OPT += -DCUDA_MPI
569  CUDA_OPT += -DCUDA_MPI
570  endif
571  ifeq ($(CUDA_MPI),y)
572  OPT += -DCUDA_MPI
573  CUDA_OPT += -DCUDA_MPI
574  endif
575 else
576  OPT := $(OPT)
577 endif
578 
579 ifeq ($(TIME_MPI),y)
580  OPT := -DTIME_MPI $(OPT)
581  CUDA_OPT += -DTIME_MPI
582 endif
583 
584 # Equilibrium dimension related OBJs
585 ifeq ($(ICONFIG), TOROIDAL3D)
586  EQ_OBJ := VMEC_interface.o
587 else
588 ifeq ($(NETCDF),y)
589  EQ_OBJ := M3DC1_interface.o
590 endif
591 endif
592 
593 CMP := mpifort
594 ##################################################################
595 # We add ".F90" to the list of suffixes to allow source files on which the
596 # co-processor will be run automatically.
597 .SUFFIXES: .o .F90 .F
598 .PHONY: clean
599 
600 # List of all the object files needed to build the code
601 OBJ:=simple_error_handler.o module.o function.o boundary_decay.o main.o setup.o restart.o\
602  diagnosis.o snapshot.o phishot.o poisson.o smooth.o field.o field_operators.o\
603  bparasolver.o tracking.o dataout3d.o pushfield.o eqdata.o eqplot.o collision.o\
604  linsolver.o $(PETSC_solver) $(AMGX_solver) $(HYPRE_solver) fft_gl.o load.o push.o\
605  shift.o charge.o pressure_push.o radial_bin.o $(CUDA_OBJ) antenna.o utility.o ohm_law.o\
606  physical_meaning.o $(EQ_OBJ)
607 
608 ifeq ($(ADIOS2), y)
609  OBJ += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o m_dataout3d_adios.o
610 endif
611 
612 HYBRID_OBJ:=hybridChargeParticle.o hybridPushParticle.o
613 KINETIC_OBJ:=gkChargeParticle.o gkPushParticle.o
614 FKI_OBJ:=fkChargeParticle.o PushFKI.o
615 
616 OBJ += $(HYBRID_OBJ) $(KINETIC_OBJ) $(FKI_OBJ)
617 
618 ifeq ($(SDP_OUT),y)
619  OBJ += SDP_interface.o
620 endif
621 
622 # set dependencies on modules based on switches
623 DEPEND_MOD :=
624 ifeq ($(SDP_OUT),y)
625  DEPEND_MOD += SDP_interface.o
626 endif
627 
628 # set commonly depended modules
629 COMMON_MOD := module.o utility.o simple_error_handler.o
630 ifeq ($(ADIOS2), y)
631  COMMON_MOD += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o m_dataout3d_adios.o
632 endif
633 
634 # Start building the executive
635 $(CMD): $(OBJ)
636  $(CMP) $(OPT) -o $(CMD) $(OBJ) $(LD_LIB)
637 
638 $(filter-out module.o,$(OBJ)): module.o
639 
640 simple_error_handler.o : simple_error_handler.F90
641  $(CMP) $(OPT) $(LIB) -c simple_error_handler.F90
642 
643 module.o : module.F90
644  $(CMP) $(OPT) $(LIB) -c module.F90
645 
646 utility.o : utility.F90 module.o
647  $(CMP) $(OPT) $(LIB) -c utility.F90
648 
649 adios2_common.o : module.o simple_error_handler.o adios2_common.F90
650  $(CMP) $(OPT) $(LIB) -c adios2_common.F90
651 
652 adios2_history_diagnostics.o : adios2_common.o module.o adios2_history_diagnostics.F90
653  $(CMP) $(OPT) $(LIB) -c adios2_history_diagnostics.F90
654 
655 adios2_data1d_diagnostics.o : adios2_common.o module.o adios2_data1d_diagnostics.F90
656  $(CMP) $(OPT) $(LIB) -c adios2_data1d_diagnostics.F90
657 
658 m_dataout3d_adios.o : adios2_common.o module.o simple_error_handler.o m_dataout3d_adios.F90
659  $(CMP) $(OPT) $(LIB) -c m_dataout3d_adios.F90
660 
661 main.o : main.F90 $(COMMON_MOD) diagnosis.o antenna.o
662  $(CMP) $(OPT) $(LIB) -c main.F90
663 
664 function.o: function.F90 $(COMMON_MOD)
665  $(CMP) $(OPT) $(LIB) -c function.F90
666 
667 boundary_decay.o : boundary_decay.F90 $(COMMON_MOD) function.o
668  $(CMP) $(OPT) $(LIB) -c boundary_decay.F90
669 
670 antenna.o: antenna.F90 $(COMMON_MOD)
671  $(CMP) $(OPT) $(LIB) -c antenna.F90
672 
673 ifeq ($(NETCDF),y)
674 M3DC1_interface.o: M3DC1_interface.F90 $(COMMON_MOD) function.o boundary_decay.o
675  $(CMP) $(OPT) $(LIB) -c M3DC1_interface.F90
676 endif
677 
678 VMEC_interface.o: VMEC_interface.F90 $(COMMON_MOD) function.o
679  $(CMP) $(OPT) $(LIB) -c VMEC_interface.F90
680 
681 eqdata.o: eqdata.F90 $(EQ_OBJ) $(COMMON_MOD) function.o boundary_decay.o $(DEPEND_MOD)
682  $(CMP) $(OPT) $(LIB) -c eqdata.F90
683 
684 eqplot.o: eqplot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD)
685  $(CMP) $(OPT) $(LIB) -c eqplot.F90
686 
687 snapshot.o: snapshot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD) physical_meaning.o
688  $(CMP) $(OPT) $(LIB) -c snapshot.F90
689 
690 phishot.o: phishot.F90 $(COMMON_MOD)
691  $(CMP) $(OPT) $(LIB) -c phishot.F90
692 
693 dataout3d.o: dataout3d.F90 $(COMMON_MOD) function.o
694  $(CMP) $(OPT) $(LIB) -c dataout3d.F90
695 
696 SDP_interface.o: SDP_interface.F90 $(COMMON_MOD) function.o
697  $(CMP) $(OPT) $(LIB) -c SDP_interface.F90
698 
699 linsolver.o: linsolver.F90 $(COMMON_MOD) poisson.o ohm_law.o bparasolver.o
700  $(CMP) $(OPT) $(LIB) -c linsolver.F90
701 
702 poisson.o: poisson.F90 $(COMMON_MOD) function.o antenna.o
703  $(CMP) $(OPT) $(LIB) -c poisson.F90
704 
705 pushfield.o: pushfield.F90 $(COMMON_MOD) function.o antenna.o
706  $(CMP) $(OPT) $(LIB) -c pushfield.F90
707 
708 ohm_law.o: ohm_law.F90 $(COMMON_MOD) function.o
709  $(CMP) $(OPT) $(LIB) -c ohm_law.F90
710 
711 field.o: field.F90 $(COMMON_MOD) antenna.o
712  $(CMP) $(OPT) $(LIB) -c field.F90
713 
714 field_operators.o: field_operators.F90 $(COMMON_MOD) antenna.o
715  $(CMP) $(OPT) $(LIB) -c field_operators.F90
716 
717 setup.o: setup.F90 $(COMMON_MOD) function.o boundary_decay.o antenna.o $(EQ_OBJ) diagnosis.o physical_meaning.o
718  $(CMP) $(OPT) $(LIB) -c setup.F90
719 
720 smooth.o: smooth.F90 $(COMMON_MOD) boundary_decay.o
721  $(CMP) $(OPT) $(LIB) -c smooth.F90
722 
723 load.o: load.F90 $(COMMON_MOD) function.o antenna.o
724  $(CMP) $(OPT) $(LIB) -c load.F90
725 
726 tracking.o: tracking.F90 $(COMMON_MOD) function.o
727  $(CMP) $(OPT) $(LIB) -c tracking.F90
728 
729 #json_module.o: json_module.F90
730 # $(CMP) $(OPT) $(LIB) -c json_module.F90
731 
732 hybrid%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
733  $(FCMP) $(OPT) $(LIB) -D_hybrid $(PREPROCESS_OPT) $*.F90 > $@
734  mv $*.i $@ || true
735 
736 gk%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
737  $(FCMP) $(OPT) $(LIB) $(PREPROCESS_OPT) $*.F90 > $@
738  mv $*.i $@ || true
739 
740 fk%.F90 : $(COMMON_MOD) boundary_decay.o function.o ChargeParticle.F90
741  $(FCMP) $(OPT) $(LIB) -D_fk $(PREPROCESS_OPT) $*.F90 > $@
742  mv $*.i $@ || true
743 
744 PushFKI.o: PushFKI.F90 $(COMMON_MOD) boundary_decay.o function.o
745  $(CMP) $(OPT) $(LIB) -c PushFKI.F90
746 
747 shift_cuda.o : shift_cuda.cu shift_cuda.cuh
748  $(NVCC) $(CUDA_OPT) -c shift_cuda.cu
749 
750 cufft.o : cufft.cu
751  $(NVCC) $(CUDA_OPT) -c cufft.cu
752 
753 amgx.o: amgx.cu
754  $(NVCC) $(CUDA_OPT) -c amgx.cu $(AMGX_LIB) $(AMGX_OPT)
755 
756 hypre.o: hypre.c
757  $(CCMP) $(COPT) -c hypre.c $(HYPRE_LIB)
758 
759 diagnosis.o: diagnosis.F90 $(COMMON_MOD) function.o antenna.o
760  $(CMP) $(OPT) $(LIB) -c diagnosis.F90
761 
762 shift.o: shift.F90 $(COMMON_MOD)
763  $(CMP) $(OPT) $(LIB) -c shift.F90
764 
765 restart.o: restart.F90 $(COMMON_MOD)
766  $(CMP) $(OPT) $(LIB) -c restart.F90
767 
768 physical_meaning.o : $(COMMON_MOD)
769  $(CMP) $(OPT) $(LIB) -c physical_meaning.F90
770 
771 .F90.o : $(COMMON_MOD)
772  $(CMP) $(OPT) $(LIB) -c $<
773 
774 # The following tag is meant to "clean" the directory by removing the
775 # executable along with all the object files created by the compilation
776 # One only has to run: gmake clean
777 
778 clean::
779  rm -f $(CMD) $(OBJ) *.mod *.o *genmod.f90 *.optrpt *.json pgipar* *.dbg *.smod
780 
781 print-%: ; @echo $* = $($*)
782 
783 tags:
784  find . -name "*.F90" -o -name "*.c" -o -name "*.cu" -o -name "*.h??" | etags -
785 
786 all: ; $(info $$COMPILER is [${COMPILER}]) $(info $$CMD is [${CMD}]) $(info $$OBJ is [${OBJ}]) $(info $$CMP is [${CMP}]) $(info $$OPT is [${OPT}]) $(info $$LIB is [${LIB}]) $(info $$LD_LIB is [${LD_LIB}])
787