r410 psmn.ens-lyon

Information about the machine https://www.psmn.ens-lyon.fr.

Instructions assume cshrc, installation under ${HOME}/softs.

Enable the basic settings:

source /usr/local/modeles/Cshrc
# The next line causes gcc 4.2.1 to be loaded from /usr/local/bin: crazy!
# source /usr/local/modeles/openmpi-1.4.1-intel-11.1.069-mkl
# You must NOT do that:
# when building GPAW, and other apllications using /usr/bin/gcc
# remove it temporarily from ~/.tcshrc and logout/login again!

Setup the root directory:

setenv HOMESOFTS ${HOME}/softs

mkdir -p ${HOMESOFTS}
cd ${HOMESOFTS}

setenv GPAW_PLATFORM `python -c "from distutils import util, sysconfig; print(util.get_platform()+'-'+sysconfig.get_python_version())"`

Set the versions:

set nose=0.11.3
# Warning: version 1.6.0 seems inconsistent about C-, Fortran-contiguous
# https://mail.scipy.org/pipermail/numpy-discussion/2011-July/057557.html
set numpy=1.5.1
set scipy=0.9.0

set acml=4.0.1

set ase=3.5.1.2175
set gpaw=0.8.0.8092
set setups=0.8.7929

and create startup scripts:

cat <<EOF > ${HOMESOFTS}/acml-${acml}-1.cshrc
setenv LD_LIBRARY_PATH \${HOMESOFTS}/acml-${acml}/gfortran64/lib:\${LD_LIBRARY_PATH}
EOF

#cat <<EOF > ${HOMESOFTS}/acml-${acml}-1.cshrc
#setenv LD_LIBRARY_PATH /softs/acml/acml${acml}/gfortran/gfortran64/lib:\${LD_LIBRARY_PATH}
#EOF

cat <<EOF > ${HOMESOFTS}/nose-${nose}-1.cshrc
setenv PYTHONPATH \${HOMESOFTS}/nose-${nose}-1/usr/lib/python2.4/site-packages:\${PYTHONPATH}
setenv PATH \${HOMESOFTS}/nose-${nose}-1/usr/bin:\${PATH}
EOF

cat <<EOF > ${HOMESOFTS}/numpy-${numpy}-1.cshrc
setenv PYTHONPATH \${HOMESOFTS}/numpy-${numpy}-1/usr/lib64/python2.4/site-packages:\${PYTHONPATH}
setenv PATH \${HOMESOFTS}/numpy-${numpy}-1/usr/bin:\${PATH}
EOF

cat <<EOF > ${HOMESOFTS}/scipy-${scipy}-1.cshrc
setenv PYTHONPATH \${HOMESOFTS}/scipy-${scipy}-1/usr/lib64/python2.4/site-packages:\${PYTHONPATH}
setenv PATH \${HOMESOFTS}/scipy-${scipy}-1/usr/bin:\${PATH}
EOF

cat <<EOF > ${HOMESOFTS}/python-ase-${ase}.cshrc
setenv PYTHONPATH \${HOMESOFTS}/python-ase-${ase}:\${PYTHONPATH}
setenv PATH \${HOMESOFTS}/python-ase-${ase}/tools:\${PATH}
EOF

cat <<EOF > ${HOMESOFTS}/gpaw-setups-${setups}.cshrc
setenv GPAW_SETUP_PATH \${HOMESOFTS}/gpaw-setups-${setups}
EOF

cat <<EOF > ${HOMESOFTS}/gpaw-${gpaw}.cshrc
setenv GPAW_PLATFORM `python -c "from distutils import util, sysconfig; print util.get_platform()+'-'+sysconfig.get_python_version()"`
setenv GPAW_HOME \${HOMESOFTS}/gpaw-${gpaw}
setenv PYTHONPATH \${GPAW_HOME}:\${PYTHONPATH}
setenv PYTHONPATH \${GPAW_HOME}/build/lib.${GPAW_PLATFORM}:\${PYTHONPATH}
setenv PATH \${GPAW_HOME}/build/bin.${GPAW_PLATFORM}:\${PATH}
setenv PATH \${GPAW_HOME}/tools:\${PATH}
EOF

cat <<EOF > ${HOMESOFTS}/campos.cshrc
#!/bin/tcsh
#
source ${HOMESOFTS}/acml-${acml}-1.cshrc
#
if ! ($?PYTHONPATH) then
    setenv PYTHONPATH ""
endif
#
setenv HOMESOFTS \${HOME}/softs
#
source \${HOMESOFTS}/nose-${nose}-1.cshrc
source \${HOMESOFTS}/numpy-${numpy}-1.cshrc
source \${HOMESOFTS}/scipy-${scipy}-1.cshrc
#
source \${HOMESOFTS}/python-ase-${ase}.cshrc
#
source \${HOMESOFTS}/gpaw-setups-${setups}.cshrc
source \${HOMESOFTS}/gpaw-${gpaw}.cshrc
EOF

Make sure that you have the right compiler:

% which gfortran  # gfortran 4.1.2
/usr/bin/gfortran

Build nose/numpy/scipy:

wget --no-check-certificate https://downloads.sourceforge.net/project/numpy/NumPy/${numpy}/numpy-${numpy}.tar.gz
wget --no-check-certificate https://downloads.sourceforge.net/project/scipy/scipy/${scipy}/scipy-${scipy}.tar.gz
wget https://python-nose.googlecode.com/files/nose-${nose}.tar.gz
tar zxf nose-${nose}.tar.gz
tar zxf numpy-${numpy}.tar.gz
tar zxf scipy-${scipy}.tar.gz
cd nose-${nose}
python setup.py install --root=${HOMESOFTS}/nose-${nose}-1 >& install.log
cd ..

continue with:

cd numpy-${numpy}
# Warning: numpy with gfortran44 fails under gpaw-python
# ImportError: numpy.core.multiarray failed to import
# Use /usr/bin/gfortran and numpy's internal blas/lapack
sed -i "s/_lib_names = \['blas'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
sed -i "s/_lib_names = \['lapack'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
# avoid "Both g77 and gfortran runtimes linked in lapack_lite !" setting --fcompiler=gnu95
# note that this forces /usr/bin/gfortran to be used
python setup.py build --fcompiler=gnu95 >& build.log
python setup.py install --root=${HOMESOFTS}/numpy-${numpy}-1 >& install.log
cd ..
source ${HOMESOFTS}/campos.cshrc
python -c "import numpy; numpy.test()"

# scipy cannot be installed due to missing suitesparse-devel and suitesparse
#cd scipy-${scipy}
#python setup.py config_fc --fcompiler=gfortran install --root=${HOMESOFTS}/scipy-${scipy}-1 2>&1 | tee install.log
#cd ..
#python -c "import scipy; scipy.test()"

Install ASE/GPAW:

wget https://wiki.fysik.dtu.dk/ase-files/python-ase-${ase}.tar.gz
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-${gpaw}.tar.gz
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-${setups}.tar.gz
tar zxf python-ase-${ase}.tar.gz
tar zxf gpaw-${gpaw}.tar.gz
tar zxf gpaw-setups-${setups}.tar.gz
mkdir testase && cd testase && testase.py >& ../testase.log
wget https://svn.fysik.dtu.dk/projects/gpaw/trunk/doc/platforms/Linux/customize_r410_psmn.py
cd ../gpaw-${gpaw}
python setup.py --remove-default-flags --customize=../customize_r410_psmn.py build_ext >& build_ext.log

The customize_r410_psmn.py looks like:

scalapack = False

compiler = 'gcc'

extra_compile_args += [
    '-O3',
    '-funroll-all-loops',
    '-fPIC',
    ]

libraries = ['gfortran']

mpi_prefix = '/softs/openmpi-gnu/'
blas_lib_path = '/home/tjiang/softs/acml-4.0.1/gfortran64/lib/'
lapack_lib_path = blas_lib_path

library_dirs = [mpi_prefix + 'lib']
include_dirs += [mpi_prefix + 'include'] # includes alreay numpy's include

extra_link_args = [
    blas_lib_path+'libacml.a',
    lapack_lib_path+'libacml.a',
    '-Wl,-rpath=' + mpi_prefix + 'lib,'
    '-rpath=' + blas_lib_path
    ]

mpicompiler= mpi_prefix + 'bin/mpicc'
mpilinker = mpicompiler

GPAW tests gpaw-test can be submitted like this:

qsub -cwd -pe mpi8 8 run.tcsh

where run.tcsh looks like this:

#!/bin/tcsh

setenv HOSTFILE $TMPDIR/machines

setenv MPIPREFIX /softs/openmpi-gnu

source ${HOME}/softs/campos.cshrc

setenv OMP_NUM_THREADS 1

${MPIPREFIX}/bin/mpirun -v -prefix ${MPIPREFIX} -mca pls_rsh_agent "ssh" -mca btl openib,tcp,self -mca btl_tcp_if_include eth1,eth0 -np $NSLOTS `which gpaw-python` `which gpaw-test`

Only some nodes have infiniband, on those without ignore the errors:

OpenIB on host r410lin???.ens-lyon.fr was unable to find any HCAs

Please make sure that your jobs do not run multi-threaded, e.g. for a job running on dl165lin7 do from a login node:

ssh dl165lin7 ps -fL

you should see 1 in the NLWP column. Numbers higher then 1 mean multi-threaded job.

It’s convenient to customize as described on the Parallel runs page.