diff --git a/job_sge.cmd b/job_sge.cmd new file mode 100644 index 0000000000000000000000000000000000000000..7de77ec3e1c84d482c7c1cba2ef45d3e1acf1a33 --- /dev/null +++ b/job_sge.cmd @@ -0,0 +1,24 @@ +#!/bin/bash +#$ -S /bin/bash + +# --- MARQOV SGE job script for the ITPA cluster --- +# move to build directory and use qsub to submit + +# start job in current working directory +#$ -cwd + +# Pretend to require so much memory that the SGE is +# forced to spread the MPI process across nodes +#$ -l h_vmem=48G # reduce for mpi16 queue + +# Choose queue and number of MPI processes +#$-pe mpi32* 8 + +# Job name +#$-N MARQOV + +#$ -e ../error.txt +#$ -o ../output.txt + +echo "Starting MARQOV on $NSLOTS nodes!" +mpirun ./src/mpiMARQOV diff --git a/job_slurm.cmd b/job_slurm.cmd new file mode 100644 index 0000000000000000000000000000000000000000..a929f7350bbfe32388ef928a53a99d9f3433fde5 --- /dev/null +++ b/job_slurm.cmd @@ -0,0 +1,29 @@ +#!/bin/bash + +# set job name +#SBATCH -J MARQOV + +# set queue +#SBATCH -p standard + +# number of nodes +#SBATCH -N 8 + +# number of tasks +#SBATCH -n 8 + +# cpus per task +#SBATCH -c 32 + +# tasks per node (set to one since MARQOV is a hybrid code) +#SBATCH --ntasks-per-node=1 + +# set max wallclock time (2 days is hard limit on julia) +#SBATCH -t 2-00:00:00 + +# logfiles +#SBATCH -o ../output.%a.out +#SBATCH -e ../output.%a.err + +# run (from build directory) +srun ./src/mpiMARQOV diff --git a/jobitpa.cmd b/jobitpa.cmd new file mode 100644 index 0000000000000000000000000000000000000000..7de77ec3e1c84d482c7c1cba2ef45d3e1acf1a33 --- /dev/null +++ b/jobitpa.cmd @@ -0,0 +1,24 @@ +#!/bin/bash +#$ -S /bin/bash + +# --- MARQOV SGE job script for the ITPA cluster --- +# move to build directory and use qsub to submit + +# start job in current working directory +#$ -cwd + +# Pretend to require so much memory that the SGE is +# forced to spread the MPI process across nodes +#$ -l h_vmem=48G # reduce for mpi16 queue + +# Choose queue and number of MPI processes +#$-pe mpi32* 8 + +# Job name +#$-N MARQOV + +#$ -e ../error.txt +#$ -o ../output.txt + +echo "Starting MARQOV on $NSLOTS nodes!" +mpirun ./src/mpiMARQOV diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b0fdfb2e4def83a7750cb6cf4404c37894bd2cf4..6804902a31010de1019d58813946529391c2eee0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,16 +27,20 @@ set_target_properties(${target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURR endforeach(target) + +find_package(MPI) if(MPI_CXX_FOUND) -if(CMAKE_VERSION VERSION_LESS 3.10) - target_include_directories(mpiMARQOV PUBLIC ${MPI_INCLUDE_PATH}) - target_link_libraries(mpiMARQOV ${MPI_C_LIBRARIES}) -else() - target_include_directories(mpiMARQOV PUBLIC ${MPI_CXX_INCLUDE_DIRS}) - target_link_libraries(mpiMARQOV ${MPI_CXX_LIBRARIES}) - set_target_properties(mpiMARQOV PROPERTIES LINK_FLAGS " ${LINK_FLAGS} ${MPI_CXX_LINK_FLAGS}") - target_compile_options(mpiMARQOV PUBLIC ${MPI_CXX_COMPILE_OPTIONS}) - target_compile_definitions(mpiMARQOV PUBLIC ${MPI_CXX_COMPILE_DEFINITIONS}) + target_compile_options(mpiMARQOV PUBLIC -DMPIMARQOV) + if(CMAKE_VERSION VERSION_LESS 3.10) + target_include_directories(mpiMARQOV PUBLIC ${MPI_INCLUDE_PATH}) + target_link_libraries(mpiMARQOV ${MPI_CXX_LIBRARIES}) + target_compile_options(mpiMARQOV PUBLIC ${MPI_CXX_COMPILE_FLAGS}) + else() + target_include_directories(mpiMARQOV PUBLIC ${MPI_CXX_INCLUDE_DIRS}) + target_link_libraries(mpiMARQOV ${MPI_CXX_LIBRARIES}) + set_target_properties(mpiMARQOV PROPERTIES LINK_FLAGS " ${LINK_FLAGS} ${MPI_CXX_LINK_FLAGS}") + target_compile_options(mpiMARQOV PUBLIC ${MPI_CXX_COMPILE_OPTIONS}) + target_compile_definitions(mpiMARQOV PUBLIC ${MPI_CXX_COMPILE_DEFINITIONS}) endif() endif(MPI_CXX_FOUND) diff --git a/src/hamiltonian/EdwardsAndersonIsing.h b/src/hamiltonian/EdwardsAndersonIsing.h index 214a9708afe0ab3ffd7a3b9a4be4bd15a8ce9cbd..29e2f36871446fc131a44d2fb646080981978de2 100644 --- a/src/hamiltonian/EdwardsAndersonIsing.h +++ b/src/hamiltonian/EdwardsAndersonIsing.h @@ -305,7 +305,6 @@ class EdwardsAndersonIsing // ---- Hamiltonian terms ---- std::array*, 1> interactions = {new EdwardsAndersonIsing_interaction(J)}; - std::array*, 0> onsite; EdwardsAndersonIsing(double J) : J(J), name("EdwardsAndersonIsing"), diff --git a/src/libmarqov/marqovscheduler.h b/src/libmarqov/marqovscheduler.h index aa9e59d6dbad778960e57e922e60200263fcdbd5..05cb22a40425a0f70f5efce3318ede6205438ff6 100644 --- a/src/libmarqov/marqovscheduler.h +++ b/src/libmarqov/marqovscheduler.h @@ -2,7 +2,7 @@ #define MARQOVSCHEDULER_H /* MIT License * - * Copyright (c) 2020-2021 Florian Goth + * Copyright (c) 2020 - 2021 Florian Goth * fgoth@physik.uni-wuerzburg.de * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -33,6 +33,10 @@ #include "marqovqueue.h" #include "core.h" +#ifdef MPIMARQOV +#include +#include +#endif #include "util/filters.h" namespace MARQOV @@ -44,9 +48,8 @@ namespace MARQOV * @tparam Sim a fully specified Marqov type */ template - class Scheduler + class CXX11Scheduler { - private: public: /** Create a full simulation from a parameter. * @@ -212,7 +215,7 @@ namespace MARQOV * @param maxptsteps How many parallel tempering steps do we do. * @param nthreads how many threads should be used. If not specified defaults to what is reported by the OS. */ - Scheduler(int maxptsteps, uint nthreads = 0) : maxpt(maxptsteps), masterstop(false), masterwork{}, + CXX11Scheduler(int maxptsteps, uint nthreads = 0) : maxpt(maxptsteps), masterstop(false), masterwork{}, workqueue(masterwork), taskqueue(((nthreads == 0)?std::thread::hardware_concurrency():nthreads)) {} @@ -220,7 +223,7 @@ namespace MARQOV * * This frees all resources and waits until all threads have finished. */ - ~Scheduler() { + ~CXX11Scheduler() { if (!nowork() && !masterstop && (taskqueue.tasks_enqueued() > 0) ) { masterwork.wait([&]{ @@ -346,6 +349,78 @@ namespace MARQOV void exchange() {} }; +#ifndef MPIMARQOV + template + using Scheduler = CXX11Scheduler; +#else + template + class MPIScheduler + { + public: + /** This gives us the parameters of a simulation and we are responsible for setting everything up. + * It has a template parameter, but of course all used parameters have to resolve to the same underlying MarqovType. + * FIXME: It is expected that all MPI ranks execute the same code until here!!! that makes it easier to have valid data on every node... + * @param p The full set of parameters that are relevant for your Problem + * @param filter A filter that can be applied before the actual creation of MARQOV + */ + template + void createSimfromParameter(ParamType& p, Callable filter) + { + if (myrank == rrctr) + myScheduler.createSimfromParameter(p, filter); + if (myrank == MASTER) + { + rrctr = (rrctr + 1) % nr_nodes; + } + MPI_Bcast(&rrctr, 1, MPI_INT, MASTER, marqov_COMM); + } + void enqueuesim(Sim& sim) + { + if (myrank == rrctr) + myScheduler.enqueuesim(sim); + if (myrank == MASTER) + { + rrctr = (rrctr + 1) % nr_nodes; + } + MPI_Bcast(&rrctr, 1, MPI_INT, MASTER, marqov_COMM); + } + void start() + { + myScheduler.start(); + } + void waitforall() {} + /** Construct MPI Scheduler. + * + * We expect MPI to be initialized beforehand. The user should feel that he is writing MPI code. + * @param maxptsteps How many parallel tempering steps do we do + * @param nthreads how many threads should be used. If not specified defaults to what is reported by the OS. + */ + MPIScheduler(int maxptsteps, uint nthreads = 0) : rrctr(0), maxpt(maxptsteps), myScheduler(maxptsteps, nthreads) + { + int mpi_inited; + MPI_Initialized(&mpi_inited); + if (!mpi_inited) + throw("MPI not initialized!"); + MPI_Comm_dup(MPI_COMM_WORLD, &marqov_COMM); + MPI_Comm_size(marqov_COMM, &nr_nodes); + MPI_Comm_rank(marqov_COMM, &myrank); + } + ~MPIScheduler() { +// MPI_Finalize(); + } + private: + int rrctr; + MPI_Comm marqov_COMM;///< our own MPI communicator + static constexpr int MASTER = 0; + CXX11Scheduler myScheduler;//MPI starts the program parallely on every node(if properly executed), hence every node needs his CXX11scheduler + int myrank; + int nr_nodes; + int maxpt; ///< how many pt steps do we do + }; + + template + using Scheduler = MPIScheduler; +#endif /** A helper class to figure out the type of the scheduler. * * @tparam Hamiltonian the type of the Hamiltonian. diff --git a/src/libmarqov/util/filters.h b/src/libmarqov/util/filters.h index 6a544b617c9609932f24559f47d794446ca9feb2..c446af29fe6d4cd114ed05dd5da07387a3e3bfc1 100644 --- a/src/libmarqov/util/filters.h +++ b/src/libmarqov/util/filters.h @@ -32,6 +32,7 @@ std::string str_repid = std::to_string(mp.repid); std::string str_beta = "beta"+std::to_string(std::get<0>(hp)); + mp.outname = str_beta+"_"+str_repid; return p; }; diff --git a/src/main.cpp b/src/main.cpp index d1c37df1576a5c50da5b5ab2015a60b6f087615b..f968eb1840fc705e87995c05d2a74f12b542dc72 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -444,17 +444,49 @@ void selectsim() } } - - - - - -int main() +int main(int argc, char* argv[]) { std::cout<<"MARQOV Copyright (C) 2020-2021, The MARQOV Project contributors"<("mc.ini", "IO", "outdir" ); + std::string logbasedir = registry.Get("mc.ini", "IO", "logdir" ); - selectsim(); + //FIXME: NEVER DELETE USER DATA + std::string command; +#ifdef MPIMARQOV + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + if (myrank == 0) { +#endif + command = "rm -r " + outbasedir; + system(command.c_str()); + command = "rm -r " + logbasedir; + system(command.c_str()); + + makeDir(outbasedir); + makeDir(logbasedir); + +#ifdef MPIMARQOV + } +#endif + selectsim(); +#ifdef MPIMARQOV + MPI_Finalize(); +#endif }