Introduction to HPC

VASP

# My .bashrc file
export MODULEPATH=${MODULEPATH}:/home/alp514/modulefiles

[alp514.sol](1083): module show intel
-------------------------------------------------------------------
/share/Apps/share/Modules/modulefiles/toolchain/intel/16.0.3:

module-whatis    Set up Intel 16.0.3 compilers. 
conflict         pgi 
conflict         gcc 
setenv           INTEL_LICENSE_FILE /share/Apps/intel/licenses/server.lic 
setenv           IPPROOT /share/Apps/intel/compilers_and_libraries_2016.3.210/linux/ipp 
setenv           MKLROOT /share/Apps/intel/compilers_and_libraries_2016.3.210/linux/mkl 
setenv           TBBROOT /share/Apps/intel/compilers_and_libraries_2016.3.210/linux/tbb 
...
snip
...

[2017-10-30 08:40.30] ~/Workshop/2017XSEDEBootCamp/MPI/Solutions
[alp514.sol](1096): mpif90 -o laplace_f90 laplace_mpi.f90 
[2017-10-30 08:40.45] ~/Workshop/2017XSEDEBootCamp/MPI/Solutions
[alp514.sol](1097): mpicc -o laplace_c laplace_mpi.c
[2017-10-30 08:40.57] ~/Workshop/2017XSEDEBootCamp/MPI/Solutions

[alp514.sol](1080): mpif90 -show
ifort -fPIC -I/share/Apps/mvapich2/2.1/intel-16.0.3/include 
  -I/share/Apps/mvapich2/2.1/intel-16.0.3/include 
  -L/share/Apps/mvapich2/2.1/intel-16.0.3/lib 
  -lmpifort -Wl,-rpath -Wl,/share/Apps/mvapich2/2.1/intel-16.0.3/lib 
  -Wl,--enable-new-dtags -lmpi

[2017-10-30 08:47.46] ~/Workshop/2017XSEDEBootCamp/MPI/Solutions
[alp514.sol-b112](993): mpiexec -n 4 ./laplace_f90 
 Maximum iterations [100-4000]?
200
 ---------- Iteration number:          100  ---------------
( 995, 995): 63.33  ( 996, 996): 72.67  ( 997, 997): 81.40  ( 998, 998): 88.97  ( 999, 999): 94.86  (1000,1000): 98.67  
 Max error at iteration          200  was   0.177397842364442     
 Total time was   0.1172750      seconds.
 ---------- Iteration number:          200  ---------------
( 995, 995): 79.11  ( 996, 996): 84.86  ( 997, 997): 89.91  ( 998, 998): 94.10  ( 999, 999): 97.26  (1000,1000): 99.28  
[2017-10-30 08:47.57] ~/Workshop/2017XSEDEBootCamp/MPI/Solutions
[alp514.sol-b112](994): mpiexec -n 4 ./laplace_c
Maximum iterations [100-4000]?
200
---------- Iteration number: 100 ------------
[995,995]: 63.33  [996,996]: 72.67  [997,997]: 81.40  [998,998]: 88.97  [999,999]: 94.86  [1000,1000]: 98.67  
---------- Iteration number: 200 ------------
[995,995]: 79.11  [996,996]: 84.86  [997,997]: 89.91  [998,998]: 94.10  [999,999]: 97.26  [1000,1000]: 99.28  

Max error at iteration 200 was 0.177398
Total time was 0.259370 seconds.

[2017-10-30 08:49.05] ~
[alp514.sol-b112](1000): srun $(which lammps) -in in.lj -var n 1 -var x 1 
LAMMPS (14 May 2016)
Lattice spacing in x,y,z = 1.6796 1.6796 1.6796
Created orthogonal box = (0 0 0) to (33.5919 1.6796 1.6796)
  24 by 1 by 1 MPI processor grid
... snipped ...

ifeq ($(COMP),gnu)
        CC = gcc
        FC = gfortran
        CFLAGS = -cpp
        OFLAGS = -fopenmp
        BINF = pif pif_omp pif_ompr
        BINC = pic pic_omp pic_ompr
else ifeq ($(COMP),intel)
        CC = icc
        FC = ifort
        CFLAGS = -fpp
        OFLAGS = -qopenmp
        BINF = pif pif_omp pif_ompr
        BINC = pic pic_omp pic_ompr
else
        CC = pgcc
        FC = pgf90
        CFLAGS = -Mpreprocess
        OFLAGS = -mp
        AFLAGS = -acc -Minfo=accel -ta=tesla:cc60 -Mcuda=kepler+
        BINF = pif pif_omp pif_ompr pif_acc
        BINC = pic pic_omp pic_ompr pic_acc
endif

ifeq ($(precision),single)
        PREC = -Mpreprocess
else
        PREC = -DDP -Mpreprocess
endif

all: $(BINC) $(BINF)
pic:
        $(CC) $(CFLAGS) $(PREC) -o pic pi_serial.c
pif:
        $(FC) $(CFLAGS) $(PREC) -o pif pi_serial.f90
pic_ompr:
        $(CC) $(CFLAGS) $(PREC) $(OFLAGS) -o pic_ompr pi_ompr.c
pic_omp:
        $(CC) $(CFLAGS) $(PREC) $(OFLAGS) -o pic_omp pi_omp.c
pif_ompr:
        $(FC) $(CFLAGS) $(PREC) $(OFLAGS) -o pif_ompr pi_ompr.f90
pif_omp:
        $(FC) $(CFLAGS) $(PREC) $(OFLAGS) -o pif_omp pi_omp.f90
pic_acc:
        $(CC) $(CFLAGS) $(PREC) $(AFLAGS) -o pic_acc pi_acc.c
pif_acc:
        $(FC) $(CFLAGS) $(PREC) $(AFLAGS) -o pif_acc pi_acc.f90
clean:
        rm -rf $(BINC) $(BINF) *~

#!/bin/bash
#PBS -q smp
#PBS -l walltime=1:00:00
#PBS -l nodes=1:ppn=1
#PBS -l mem=4GB
#PBS -N myjob

cd ${PBS_O_WORKDIR}
./myjob < filename.in > filename.out

#!/bin/bash
#SBATCH --partition=lts
#SBATCH --time=1:00:00
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --job-name myjob

cd ${SLURM_SUBMIT_DIR}
./myjob < filename.in > filename.out

#!/bin/bash
#SBATCH --partition=lts
#SBATCH --time=1:00:00
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=20
## For --partition=imlab, 
###  use --ntasks-per-node=22
### and --qos=nogpu
#SBATCH --job-name myjob

module load mvapich2

cd ${SLURM_SUBMIT_DIR}
srun ./myjob < filename.in > filename.out

exit

#!/bin/bash
#PBS -q normal
#PBS -l walltime=1:00:00
#PBS -l nodes=2:ppn=16
#PBS -N myjob

module load openmpi/1.6.5/pgi/13.10

cd ${PBS_O_WORKDIR}

mpiexec -n $PBS_PPN -f $PBS_NODEFILE ./myjob \
   < filename.in > filename.out

exit

#!/bin/tcsh
#SBATCH --partition=imlab
# Directives can be combined on one line
#SBATCH --time=1:00:00 --nodes=1 --ntasks-per-node=22
#SBATCH --qos=nogpu
#SBATCH --job-name myjob

cd ${SLURM_SUBMIT_DIR}
# Use either
setenv OMP_NUM_THREADS 22
./myjob < filename.in > filename.out

# OR
OMP_NUM_THREADS=22 ./myjob < filename.in > filename.out

exit

#!/bin/bash
#PBS -q normal
#PBS -l walltime=1:00:00
#PBS -l nodes=1:ppn=32
#PBS -N myjob

cd ${PBS_O_WORKDIR}
export OMP_NUM_THREADS=32
./myjob < filename.in > filename.out

exit

#!/bin/tcsh
#SBATCH --partition=imlab
# Directives can be combined on one line
#SBATCH --time=1:00:00
#SBATCH --nodes=1
# 1 CPU can be be paired with only 1 GPU
# 1 GPU can be paired with all 24 CPUs
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
# Need both GPUs, use --gres=gpu:2
#SBATCH --job-name myjob

cd ${SLURM_SUBMIT_DIR}
# Load LAMMPS Module
module load lammps/17nov16-gpu
# Run LAMMPS for input file in.lj
srun $(which lammps) -in in.lj -sf gpu -pk gpu 1 gpuID ${CUDA_VISIBLE_DEVICE}

exit

Cluster	Cores	CPU Memory	CPU TFLOPs	GPUs	CUDA Cores	GPU Memory	GPU TFLOPS
Maia	32	128	0.640
Monocacy	128	512	2.662
Unnamed	320	1280	6.656
Baltrusaitislab	96	384	1.766
Pisces	120	384	4.416	6	29952	144	17.472
Sol	1300	6720	47.366	110	281600	880	28.27
Total	1996	9408	63.507	116	311552	1024	45.742

Command	Description
`module avail`	show list of software available on resource
`module load abc`	add software `abc` to your environment (modify your `PATH`, `LD_LIBRARY_PATH` etc as needed)
`module unload abc`	remove `abc` from your environment
`module swap abc1 abc2`	swap `abc1` with `abc2` in your environment
`module purge`	remove all modules from your environment
`module show abc`	display what variables are added or modified in your environment
`module help abc`	display help message for the module `abc`

Language	GNU	Intel	PGI
Fortran	gfortran	ifort	pgfortran
C	gcc	icc	pgcc
C++	g++	icpc	pgc++

Compiler	OpenMP Flag	TBB Flag
GNU	-fopenmp	-L$TBBROOT/lib/intel64_lin/gcc4.4 -ltbb
Intel	-qopenmp	-L$TBBROOT/lib/intel64_lin/gcc4.4 -ltbb
PGI	-mp

Language	Compile Command
Fortran	mpif90
C	mpicc
C++	mpicxx

Option Description	mpiexec	mpirun	mpirun_rsh
run on `x` cores	-n x	-np x	-n x
location of the hostfile	-f filename	-machinefile filename	-hostfile filename

Cluster	Partition Name	Max Runtime in hours	Max SU consumed node per hour
Sol	lts	72	20 (will change to 18+2)
	imlab	48	22
	imlab-gpu	48	24
	eng	72	24 (will change to 22+2)
	engc	72	24 (will change to 22+2)
	all-cpu	48
	all-gpu	48

SLURM Directive	Description
#SBATCH --partition=queuename	Submit job to the queuename partition.
#SBATCH --time=hh:mm:ss	Request resources to run job for hh hours, mm minutes and ss seconds.
#SBATCH --nodes=m	Request resources to run job on m nodes.
#SBATCH --ntasks-per-node=n	Request resources to run job on n processors on each node requested.
#SBATCH --ntasks=n	Request resources to run job on a total of n processors.
#SBATCH --job-name=jobname	Provide a name, jobname to your job.
#SBATCH --output=filename.out	Write SLURM standard output to file filename.out.
#SBATCH --error=filename.err	Write SLURM standard error to file filename.err.
#SBATCH --mail-type=events	Send an email after job status events is reached.
	events can be NONE, BEGIN, END, FAIL, REQUEUE, ALL, TIME_LIMIT(_90,80)
#SBATCH --mail-user=address	Address to send email.
#SBATCH --account=mypi	charge job to the mypi account

SLURM Directive	Description
#SBATCH --qos=nogpu	Request a quality of service (qos) for the job in `imlab`, `engc` partitions.
	Job will remain in queue indefinitely if you do not specify qos
#SBATCH --gres=gpu:#	Specifies number of gpus requested in the gpu partitions
	You can request 1 or 2 gpus with a minimum of 1 core or cpu per gpu

Long Form	Short Form
--partition=queuename	-p queuename
--time=hh:mm:ss	-t hh:mm:ss
--nodes=m	-N m
--ntasks=n	-n n
--account=mypi	-A mypi
--job-name=jobname	-J jobname
--output=filename.out	-o filename.out

Pattern	Description
%A	Job array's master job allocation number.
%a	Job array ID (index) number.
%J	jobid.stepid of the running job. (e.g. "128.0")
%j	jobid of the running job.
%N	short hostname. This will create a separate IO file per node.
%n	Node identifier relative to current job (e.g. "0" is the first node of the running job) This will create a separate IO file per node.
%s	stepid of the running job.
%t	task identifier (rank) relative to current job. This will create a separate IO file per task.
%u	User name.
%x	Job name.

PBS Directive	Description
#PBS -q queuename	Submit job to the queuename queue.
#PBS -l walltime=hh:mm:ss	Request resources to run job for hh hours, mm minutes and ss seconds.
#PBS -l nodes=m:ppn=n	Request resources to run job on n processors each on m nodes.
#PBS -l mem=xGB	Request xGB per node requested, applicable on Maia only
#PBS -N jobname	Provide a name, jobname to your job.
#PBS -o filename.out	Write PBS standard output to file filename.out.
#PBS -e filename.err	Write PBS standard error to file filename.err.
#PBS -j oe	Combine PBS standard output and error to the same file.
#PBS -M your email address	Address to send email.
#PBS -m status	Send an email after job status status is reached.
	status can be a (abort), b (begin) or e (end). The arguments can be combined
	for e.g. abe will send email when job begins and either aborts or ends

SLURM Command	Description	PBS Command
SLURM_SUBMIT_DIR	Directory where the `qsub` command was executed	PBS_O_WORKDIR
SLURM_JOB_NODELIST	Name of the file that contains a list of the HOSTS provided for the job	PBS_NODEFILE
SLURM_NTASKS	Total number of cores for job	PBS_NP
SLURM_JOBID	Job ID number given to this job	PBS_JOBID
SLURM_JOB_PARTITION	Queue job is running in	PBS_QUEUE
	Walltime in secs requested	PBS_WALLTIME
	Name of the job. This can be set using the -N option in the PBS script	PBS_JOBNAME
	Indicates job type, PBS_BATCH or PBS_INTERACTIVE	PBS_ENVIRONMENT
	value of the SHELL variable in the environment in which qsub was executed	PBS_O_SHELL
	Home directory of the user running qsub	PBS_O_HOME

SLURM Command	Description	PBS Command
squeue	check job status (all jobs)	qstat
squeue -u username	check job status of user username	qstat -u username
squeue --start	Show estimated start time of jobs in queue	showstart jobid
scontrol show job jobid	Check status of your job identified by jobid	checkjob jobid
scancel jobid	Cancel your job identified by jobid	qdel jobid
scontrol hold jobid	Put your job identified by jobid on hold	qhold jobid
scontrol release jobid	Release the hold that you put on jobid	qrls jobid

Cluster	Partition	Max Memory/core (GB)	Recommended Memory/Core (GB)
Sol	lts	6.4	6.2
	eng/imlab/imlab-gpu	5.3	5.1
	engc	2.66	2.4

Introduction to HPC

Library & Technology Services

About Us?

What do we do?

Training & Consulting

Full Day Workshops

Research Computing Resources

Research Computing Resources

LTS Managed Faculty Resources

Total Computational Resources Supported

Apply for an account

Allocation Charges - Effective Oct. 1, 2016

Condo Investments

Condo Investors

What about Storage resources

Network Layout Sol & Ceph Storage Cluster

Accessing Research Computing Resources

Available Software

Installed Software

More Software

More Software

Module Command

Creating your own modules

Module File Example

Compilers

Compiling Code

Compilers for Parallel Programming: OpenMP & TBB

Compilers for Parallel Programming: MPI

MPI Libraries

Running MPI Programs

Running MPI Codes

Compiling Using Makefiles

Makefile Examples

Cluster Environment

How to run jobs

Scheduler & Resource Management

Job Scheduling

Backfilling

How much time must I request

Available Queues

How much memory can or should I use per core?

Useful SBATCH Directives

Useful SBATCH Directives (contd)

SBATCH Filename Patterns

Useful PBS Directives

Useful PBS/SLURM environmental variables

Basic Job Manager Commands

Job Types

Job Types: Interactive

Job Types: Batch

Minimal submit script for Serial Jobs

Minimal submit script for MPI Job

Minimal submit script for OpenMP Job

Minimal submit script for LAMMPS GPU job

Monitoring & Manipulating Jobs

Usage Reporting

Need to run multiple jobs in sequence?

Additional Help & Information