curie.ccc.cea.fr GPU¶

Here you find information about the system https://www-hpc.cea.fr/en/complexe/tgcc-curie.htm.

Warning: May 14 2013: rpa-gpu-expt branch fails to compile due to cublasZdgmm missing in cuda-4.2.

The system runs Bull Linux. The installation assumes bash shell:

packages are installed under ~/CAMd:
```
mkdir ~/CAMd
cd ~/CAMd
```
module files are located under ~/CAMd/modulefiles:
```
mkdir ~/CAMd/modulefiles
```

download the customize_curie_gpu.py file:

import os

scalapack = True
hdf5 = False
# ld: /usr/local/phdf5-1.8.5/lib/libhdf5.a(H5.o): relocation R_X86_64_32 against `.rodata.str1.4' can not be used when making a shared object; recompile with -fPIC
compiler = 'icc'

mpi='/opt/mpi/bullxmpi/1.1.16.5'
mkl='/usr/local/Intel_compilers/c/composer_xe_2011_sp1.7.256/mkl/lib/intel64'
intel='/usr/local/Intel_compilers/c/composer_xe_2011_sp1.7.256/compiler/lib/intel64'
hdf='/usr/local/phdf5-1.8.5'
#
# cublasZdgmm does not exist in cuda 4.2
# /tmp/ipo_iccjq2M5h1.o: In function `cudgmm':
# ipo_out1.c:(.text.hot0001d+0x522b): undefined reference to `cublasZdgmm'
# strings /usr/local/cuda-4.2/lib64/libcublas.so | grep "cublasZdgmm"
cuda='/usr/local/cuda-4.2'  # comment out if no cuda

libraries =[
    'cublas', 'cufft', 'cuda',  # comment out if no cuda
    'cudart',  # comment out if no cuda
    #'mkl_def',
    'mkl_scalapack_lp64', 'mkl_intel_lp64', 'mkl_sequential',
    'mkl_core', 'mkl_blacs_openmpi_lp64',
    #'hdf5',
    'mpi',
    ]
library_dirs =[
    intel,
    os.path.join(mpi, 'lib'),
    mkl,
    os.path.join(cuda, 'lib64'),  # comment out if no cuda
    #os.path.join(hdf, 'lib'),
    ]
include_dirs +=[
    os.path.join(mpi, 'include'),
    os.path.join(cuda, 'include'),  # comment out if no cuda
    #os.path.join(hdf, 'include'),
    ]
extra_link_args =[
    '-Wl,-rpath=' + intel +
    ',-rpath=' + os.path.join(mpi, 'lib') +
    ',-rpath=' + os.path.join(cuda, 'lib64') +  # comment out if no cuda
    ',-rpath=' + mkl
    #',-rpath=' + os.path.join(hdf, 'lib')
    ]
extra_compile_args =['-xHOST', '-O3', '-ipo', '-std=c99', '-fPIC', '-Wall']
extra_objects += ['./c/cukernels.o']
define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
mpicompiler = os.path.join(mpi, 'bin', 'mpicc')
mpilinker = mpicompiler

download packages with download_curie_gpu.sh:

export APPS="readlink -f ~/CAMd"
export MODULEFILES="${APPS}/modulefiles"

# warning - firewall blocks that, so download on other machine and scp!

cd ${APPS}
# download packages
nose_version=1.1.2
wget https://pypi.python.org/packages/source/n/nose/nose-${nose_version}.tar.gz
numpy_version=1.5.1
wget https://downloads.sourceforge.net/numpy/numpy-${numpy_version}.tar.gz
scipy_version=0.9.0
wget https://downloads.sourceforge.net/project/scipy/scipy/${scipy_version}/scipy-${scipy_version}.tar.gz
ase_version=3.7.0.3168
wget https://wiki.fysik.dtu.dk/ase-files/python-ase-${ase_version}.tar.gz
gpaw_version=0.9.0.8965
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-${gpaw_version}.tar.gz
gpaw_setups_version=0.9.9672
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-${gpaw_setups_version}.tar.gz
# OK, curie does not allow svn!
svn co https://svn.fysik.dtu.dk/projects/gpaw/branches/rpa-gpu-expt

install packages, deploy modules and test with install_curie_gpu.sh:

export APPS=`readlink -f ~/CAMd`
export CAMD_MODULEFILES="${APPS}/modulefiles"

export GPAW_PLATFORM=`python -c "from distutils import util, sysconfig; print util.get_platform()+'-'+sysconfig.get_python_version()"`
export PYTHONVERSION=`python -c "from distutils import sysconfig; print sysconfig.get_python_version()"`

# build packages

nose_version=1.1.2
tar zxf nose-${nose_version}.tar.gz
cd nose-${nose_version}
python setup.py install --root=${APPS}/nose-${nose_version}-1
cd ..

mkdir -p ${CAMD_MODULEFILES}/nose
cat <<EOF > ${CAMD_MODULEFILES}/nose/${nose_version}-1
#%Module1.0
set apps_path ${APPS}
prepend-path    PATH                \$apps_path/nose-${nose_version}-1/usr/bin
prepend-path    PYTHONPATH          \$apps_path/nose-${nose_version}-1/usr/lib/python${PYTHONVERSION}/site-packages/
unset apps_path
EOF

numpy_version=1.5.1
tar zxf numpy-${numpy_version}.tar.gz
cd  numpy-${numpy_version}
# atlas on curie is built without -fPIC
# /usr/bin/ld: /usr/local/atlas-3.9.72/lib/libcblas.a(cblas_dgemm.o): relocation R_X86_64_32 against `.rodata.str1.8' can not be used when making a shared object; recompile with -fPIC
# and atlas-devel is not installed!
# so hack!
ln -s /usr/lib64/atlas/libatlas.so.3.0 libatlas.so
ln -s /usr/lib64/atlas/libcblas.so.3.0 libcblas.so
ln -s /usr/lib64/atlas/libclapack.so.3 libclapack.so
ln -s /usr/lib64/atlas/libf77blas.so.3 libf77blas.so
ln -s /usr/lib64/atlas/liblapack.so.3 liblapack.so
echo "[DEFAULT]" > site.cfg
echo "library_dirs = $PWD" >> site.cfg
echo "include_dirs = /usr/local/atlas-3.9.72/include" >> site.cfg
# avoid "Both g77 and gfortran runtimes linked in lapack_lite !" setting --fcompiler=gnu95
# note that this forces /usr/bin/gfortran to be used
python setup.py build --fcompiler=gnu95 2>&1 | tee build.log
python setup.py install --root=${APPS}/numpy-${numpy_version}-1 2>&1 | tee install.log
cd ..

mkdir -p ${CAMD_MODULEFILES}/numpy
cat <<EOF > ${CAMD_MODULEFILES}/numpy/${numpy_version}-1
#%Module1.0
set apps_path ${APPS}
prereq nose
prepend-path    PATH                \$apps_path/numpy-${numpy_version}-1/usr/bin
prepend-path    PYTHONPATH          \$apps_path/numpy-${numpy_version}-1/usr/lib64/python${PYTHONVERSION}/site-packages/
unset apps_path
EOF

# the atlas is missing on the hybrid (only?) nodes so hack again!
mkdir atlas && cd atlas
cp -p /usr/lib64/atlas/liblapack.so.3 .
cp -p /usr/lib64/atlas/libf77blas.so.3 .
cp -p /usr/lib64/atlas/libcblas.so.3 .
cp -p /usr/lib64/atlas/libatlas.so.3 .
cd ..

module use $CAMD_MODULEFILES
module load nose
module load numpy  # scipy build needs numpy!

scipy_version=0.9.0
tar zxf scipy-${scipy_version}.tar.gz
cd  scipy-${scipy_version}
# avoid g77 - leads to Segmentation faults
# note that this forces /usr/bin/gfortran to be used
python setup.py build --fcompiler=gnu95 2>&1 | tee build.log
python setup.py install --root=${APPS}/scipy-${scipy_version}-1 2>&1 | tee install.log
cd ..

mkdir -p ${CAMD_MODULEFILES}/scipy
cat <<EOF > ${CAMD_MODULEFILES}/scipy/${scipy_version}-1
#%Module1.0
set apps_path ${APPS}
prereq nose
prepend-path    PATH                \$apps_path/scipy-${scipy_version}-1/usr/bin
prepend-path    PYTHONPATH          \$apps_path/scipy-${scipy_version}-1/usr/lib64/python${PYTHONVERSION}/site-packages/
unset apps_path
EOF

ase_version=3.7.0.3168
tar zxf python-ase-${ase_version}.tar.gz

mkdir -p ${CAMD_MODULEFILES}/python-ase
cat <<EOF > ${CAMD_MODULEFILES}/python-ase/${ase_version}-1
#%Module1.0
set apps_path ${APPS}
prereq numpy
prepend-path    PATH                \$apps_path/python-ase-${ase_version}/tools
prepend-path    PYTHONPATH          \$apps_path/python-ase-${ase_version}/
unset apps_path
EOF

gpaw_setups_version=0.9.9672
tar zxf gpaw-setups-${gpaw_setups_version}.tar.gz

mkdir -p ${CAMD_MODULEFILES}/gpaw-setups
cat <<EOF > ${CAMD_MODULEFILES}/gpaw-setups/${gpaw_setups_version}-1
#%Module1.0
set apps_path ${APPS}
prepend-path    GPAW_SETUP_PATH     \$apps_path/gpaw-setups-${gpaw_setups_version}
unset apps_path
EOF

gpaw_version=0.9.0.8965
tar zxf gpaw-${gpaw_version}.tar.gz

mkdir -p ${CAMD_MODULEFILES}/gpaw
cat <<EOF > ${CAMD_MODULEFILES}/gpaw/${gpaw_version}-1
#%Module1.0
set apps_path ${APPS}
prereq python-ase
prereq gpaw-setups
prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/tools
prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/build/bin.${GPAW_PLATFORM}
prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/
prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/build/lib.${GPAW_PLATFORM}
setenv OMP_NUM_THREADS 1
unset apps_path
EOF

module load nose
module load numpy
module load scipy
# test numpy and scipy
python -c "import numpy; numpy.test()"
python -c "import scipy; scipy.test()"

# test ase
module load python-ase
mkdir -p testase
cd testase
testase.py --no-display 2>&1 | tee testase.log
cd ..

# build gpaw
cd gpaw-${gpaw_version}
module load cuda
# if on rpa-gpu-expt branch
rm -f c/cukernels.o
cd c
nvcc -arch sm_20 -c cukernels.cu -Xcompiler -fPIC
cd ..
# wget https://svn.fysik.dtu.dk/projects/gpaw/trunk/config.py  # fixed in trunk
python setup.py build_ext --customize=../customize_curie_gpu.py --remove-default-flags 2>&1 | tee build_ext.log
cd ..
module load gpaw-setups
module load gpaw

Note that every time you wish to install a new version of a package, and deploy new module file, better keep the old module file.

submit a test job:

ccc_msub msub_curie_gpu.sh

using the following msub_curie_gpu.sh:

#/bin/sh
#MSUB -n 8 # number of tasks
#MSUB -T 4600 # time
#MSUB -q hybrid # use hybrid for GPU
#MSUB -A paxxxx

set -x
cd ${BRIDGE_MSUB_PWD}
module use ${HOME}/CAMd/modulefiles
module load nose
# blas/lapack/atlas missing
export LD_LIBRARY_PATH=${HOME}/CAMd/atlas:${LD_LIBRARY_PATH}
module load numpy
module load scipy
module load python-ase
module load gpaw-setups/0.8.7929-1
#module load gpaw-setups/0.9.9672-1
module load gpaw
#ccc_mprun gpaw-python gpaw-0.9.0.8965/gpaw/test/2Al.py
ccc_mprun gpaw-python `which gpaw-test`