From 551120ac1bde95c542c524029a356d64f96cc8ce Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Mon, 2 Nov 2020 17:08:23 +0100 Subject: [PATCH 01/16] python: integrate within autotools, it's not really fully adequate --- Makefile.am | 4 ++ configure.ac | 43 +++++++++++++ starpupy/Makefile.am | 30 +++++++++ starpupy/src/Makefile.am | 61 +++++++++++++++++++ .../src/starpu/{Makefile => Makefile.old} | 2 +- starpupy/tests/Makefile.am | 29 +++++++++ starpupy/tests/{Makefile => Makefile.old} | 0 starpupy/tests/execute.sh.in | 34 +++++++++++ starpupy/tests/starpu_py.sh | 19 ++++++ starpupy/tests/starpu_py_parallel.sh | 19 ++++++ 10 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 starpupy/Makefile.am create mode 100644 starpupy/src/Makefile.am rename starpupy/src/starpu/{Makefile => Makefile.old} (85%) create mode 100644 starpupy/tests/Makefile.am rename starpupy/tests/{Makefile => Makefile.old} (100%) create mode 100755 starpupy/tests/execute.sh.in create mode 100755 starpupy/tests/starpu_py.sh create mode 100755 starpupy/tests/starpu_py_parallel.sh diff --git a/Makefile.am b/Makefile.am index 7a94252386..1bb3b11dee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -53,6 +53,10 @@ if STARPU_BUILD_STARPURM SUBDIRS += starpurm endif +if STARPU_BUILD_STARPUPY +SUBDIRS += starpupy +endif + if STARPU_BUILD_SC_HYPERVISOR SUBDIRS += sc_hypervisor endif diff --git a/configure.ac b/configure.ac index 738e6e4cbd..4e88ba7e1f 100644 --- a/configure.ac +++ b/configure.ac @@ -3394,6 +3394,40 @@ AC_ARG_ENABLE(starpurm-examples, [AS_HELP_STRING([--enable-starpurm-examples], enable_starpurm_examples=$enableval, enable_starpurm_examples=no) AM_CONDITIONAL(STARPU_BUILD_STARPURM_EXAMPLES, [test x$enable_starpurm_examples = xyes]) +##################################### +# StarPUPy # +##################################### + +starpupy_support=no +AC_ARG_ENABLE(starpupy, [AS_HELP_STRING([--enable-starpupy], [enable StarPU python interface])], + enable_starpupy=$enableval, enable_starpupy=no) +if test "x$enable_starpupy" != xno +then + starpupy_support=yes + AC_CHECK_PROGS([PYTHON_CONFIG], python3-config) + if test "$ac_cv_prog_PYTHON_CONFIG" == "" + then + AC_MSG_ERROR([python3-config missing, cannot build StarPU python interface]) + fi + AC_CHECK_PROGS([PYTHON], python3) + if test "$ac_cv_prog_PYTHON" == "" + then + AC_MSG_ERROR([python3 missing, cannot build StarPU python interface]) + fi + AC_SUBST(PYTHON) + SAVED_CPPFLAGS="${CPPFLAGS}" + CPPFLAGS="$CPPFLAGS $($ac_cv_prog_PYTHON_CONFIG --includes)" + AC_CHECK_HEADER([Python.h],[have_valid_starpupy=yes],[have_valid_starpupy=no]) + if test "$have_valid_starpupy" = "no" + then + AC_MSG_ERROR([Python.h missing, cannot build StarPU python interface]) + fi + CPPFLAGS="$SAVED_CPPFLAGS" + STARPUPY_CPPFLAGS="$($ac_cv_prog_PYTHON_CONFIG --includes)" + AC_SUBST(STARPUPY_CPPFLAGS) +fi +AM_CONDITIONAL(STARPU_BUILD_STARPUPY, [test x$starpupy_support = xyes]) + ########################################## # Documentation # ########################################## @@ -3555,6 +3589,10 @@ AC_CONFIG_COMMANDS([executable-scripts], [ test -e tools/starpu_paje_state_stats.R || ln -sf $ac_abs_top_srcdir/tools/starpu_paje_state_stats.R tools/starpu_paje_state_stats.R test -e tools/starpu_trace_state_stats.py || ln -sf $ac_abs_top_srcdir/tools/starpu_trace_state_stats.py tools/starpu_trace_state_stats.py chmod +x tools/starpu_trace_state_stats.py + chmod +x starpupy/tests/execute.sh + for x in starpupy/tests/starpu_py.sh starpupy/tests/starpu_py_parallel.sh; do + test -e $x || mkdir -p $(dirname $x) && ln -sf $ac_abs_top_srcdir/$x $(dirname $x) + done chmod +x julia/examples/execute.sh for x in julia/examples/check_deps/check_deps.sh julia/examples/mult/mult_starpu.sh julia/examples/mult/perf.sh julia/examples/variable/variable.sh julia/examples/task_insert_color/task_insert_color.sh julia/examples/vector_scal/vector_scal.sh julia/examples/mandelbrot/mandelbrot.sh julia/examples/callback/callback.sh julia/examples/dependency/task_dep.sh julia/examples/dependency/tag_dep.sh julia/examples/dependency/end_dep.sh julia/examples/axpy/axpy.sh julia/examples/gemm/gemm.sh julia/examples/cholesky/cholesky.sh; do test -e $x || mkdir -p $(dirname $x) && ln -sf $ac_abs_top_srcdir/$x $(dirname $x) @@ -3624,6 +3662,10 @@ AC_OUTPUT([ starpurm/src/Makefile starpurm/examples/Makefile starpurm/starpurm-1.3.pc + starpupy/Makefile + starpupy/src/Makefile + starpupy/tests/Makefile + starpupy/tests/execute.sh examples/Makefile examples/stencil/Makefile tests/Makefile @@ -3698,6 +3740,7 @@ AC_MSG_NOTICE([ Master-Slave MPI enabled: $use_mpi_master_slave FFT Support: $fft_support Resource Management enabled: $starpurm_support + Python Interface enabled: $starpupy_support OpenMP runtime support enabled: $enable_openmp Cluster support enabled: $enable_cluster SOCL enabled: $build_socl diff --git a/starpupy/Makefile.am b/starpupy/Makefile.am new file mode 100644 index 0000000000..9304706c7d --- /dev/null +++ b/starpupy/Makefile.am @@ -0,0 +1,30 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +include $(top_srcdir)/starpu-subdirtests.mk + +SUBDIRS = src +SUBDIRS += tests + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = starpupy-1.3.pc + +#versincludedir = $(includedir)/starpu/$(STARPU_EFFECTIVE_VERSION) +#versinclude_HEADERS = \ +# include/starpurm.h + +#nodist_versinclude_HEADERS = \ +# include/starpurm_config.h diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am new file mode 100644 index 0000000000..83ba45e31e --- /dev/null +++ b/starpupy/src/Makefile.am @@ -0,0 +1,61 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +include $(top_srcdir)/starpu-notests.mk + +SUBDIRS = + +#CLEANFILES = *.gcno *.gcda *.linkinfo + +AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include +AM_CFLAGS = -Wall -g $(STARPU_CUDA_CPPFLAGS) $(STARPU_OPENCL_CPPFLAGS) $(FXT_CFLAGS) $(MAGMA_CFLAGS) $(HWLOC_CFLAGS) $(GLOBAL_AM_CFLAGS) $(NMAD_CFLAGS) $(STARPUPY_CPPFLAGS) +AM_LDFLAGS = $(STARPU_OPENCL_LDFLAGS) $(STARPU_CUDA_LDFLAGS) $(STARPU_COI_LDFLAGS) $(STARPU_SCIF_LDFLAGS) $(NMAD_LDFLAGS) +#AM_LIBADD = + +PYTHON ?= python3 +AM_CPPFLAGS += $(shell $(PYTHON)-config --includes) -Wall -O2 -g + +all: starpu/starpupy.so + for f in $(top_srcdir)/starpupy/src/starpu/*py ; do cp $$f starpu ; done +starpu/starpupy.so: starpu/starpu_task_wrapper.c + $(MKDIR_P) starpu + $(CC) -fPIC $(AM_CFLAGS) $< -o $@ -shared $(AM_CPPFLAGS) $(AM_LDFLAGS) -L../../src/.libs -lstarpu-1.3 + +CLEANFILES= starpu/starpupy.so + +#libstarpupy_so_version = $(LIBSTARPUPY_INTERFACE_CURRENT):$(LIBSTARPUPY_INTERFACE_REVISION):$(LIBSTARPUPY_INTERFACE_AGE) +# +##lib_LTLIBRARIES = libstarpupy-@STARPU_EFFECTIVE_VERSION@.la +#lib_LTLIBRARIES = libstarpupy.la +# +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CPPFLAGS = $(AM_CPPFLAGS) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CFLAGS = $(AM_CFLAGS) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) +# +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = \ +# starpu_task_wrapper.c +# +#libstarpupy_la_CPPFLAGS = $(AM_CPPFLAGS) +#libstarpupy_la_CFLAGS = $(AM_CFLAGS) +#libstarpupy_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) +#libstarpupy_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) +# +#libstarpupy_la_SOURCES = \ +# starpu_task_wrapper.c + +#noinst_HEADERS = \ +# starpurm_private.h diff --git a/starpupy/src/starpu/Makefile b/starpupy/src/starpu/Makefile.old similarity index 85% rename from starpupy/src/starpu/Makefile rename to starpupy/src/starpu/Makefile.old index a65543b86f..0080f7d827 100644 --- a/starpupy/src/starpu/Makefile +++ b/starpupy/src/starpu/Makefile.old @@ -6,7 +6,7 @@ LDLIBS += $(shell pkg-config --libs starpu-1.3) all: starpupy.so -starpupy.so: starpu_task_wrapper.c Makefile +starpupy.so: starpu_task_wrapper.c Makefile.old $(CC) -fPIC $(CFLAGS) $< -o $@ -shared $(CPPFLAGS) $(LDLIBS) clean: diff --git a/starpupy/tests/Makefile.am b/starpupy/tests/Makefile.am new file mode 100644 index 0000000000..105afb2509 --- /dev/null +++ b/starpupy/tests/Makefile.am @@ -0,0 +1,29 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# +SUBDIRS = + +CLEANFILES = *.gcno *.gcda *.linkinfo + +#AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src -I$(top_builddir)/src -I$(top_builddir)/include +#AM_CPPFLAGS += -I$(top_srcdir)/starpurm/include -I$(top_srcdir)/starpurm/src -I$(top_builddir)/starpurm/src -I$(top_builddir)/starpurm/include +#AM_CFLAGS = -Wall -g $(HWLOC_CFLAGS) $(DLB_CFLAGS) +#LDADD = $(top_builddir)/starpurm/src/libstarpurm-@STARPU_EFFECTIVE_VERSION@.la $(top_builddir)/src/@LIBSTARPU_LINK@ $(HWLOC_LIBS) $(DLB_LIBS) + +examplebindir = $(libdir)/starpupy/examples + +TESTS = +TESTS += starpu_py.sh +TESTS += starpu_py_parallel.sh diff --git a/starpupy/tests/Makefile b/starpupy/tests/Makefile.old similarity index 100% rename from starpupy/tests/Makefile rename to starpupy/tests/Makefile.old diff --git a/starpupy/tests/execute.sh.in b/starpupy/tests/execute.sh.in new file mode 100755 index 0000000000..01757c8cac --- /dev/null +++ b/starpupy/tests/execute.sh.in @@ -0,0 +1,34 @@ +#!@REALBASH@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +set -x + +tstdir=@STARPU_SRC_DIR@/starpupy/tests + +modpath=@STARPU_BUILD_DIR@/src/.libs: +pypath=@STARPU_BUILD_DIR@/starpupy/src:$PYTHONPATH + +tstfile=$1 +if test ! -f $tstdir/$tstfile +then + echo "Error. File $tstdir/$tstfile not found" + exit 1 +fi +shift + +PYTHONPATH=$pypath LD_LIBRARY_PATH=$modpath @PYTHON@ $tstdir/$tstfile $* + diff --git a/starpupy/tests/starpu_py.sh b/starpupy/tests/starpu_py.sh new file mode 100755 index 0000000000..df367755d3 --- /dev/null +++ b/starpupy/tests/starpu_py.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +$(dirname $0)/execute.sh starpu_py.py + diff --git a/starpupy/tests/starpu_py_parallel.sh b/starpupy/tests/starpu_py_parallel.sh new file mode 100755 index 0000000000..601543c38e --- /dev/null +++ b/starpupy/tests/starpu_py_parallel.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +$(dirname $0)/execute.sh starpu_py_parallel.py + -- GitLab From c696417db013ca9b11e4d82095934afd781bfaf8 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Mon, 2 Nov 2020 17:08:23 +0100 Subject: [PATCH 02/16] python: integrate within autotools, it's not really fully adequate --- Makefile.am | 4 ++ configure.ac | 43 +++++++++++++ starpupy/Makefile.am | 30 +++++++++ starpupy/src/Makefile.am | 61 +++++++++++++++++++ .../src/starpu/{Makefile => Makefile.old} | 2 +- starpupy/tests/Makefile.am | 29 +++++++++ starpupy/tests/{Makefile => Makefile.old} | 0 starpupy/tests/execute.sh.in | 34 +++++++++++ starpupy/tests/starpu_py.sh | 19 ++++++ starpupy/tests/starpu_py_parallel.sh | 19 ++++++ 10 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 starpupy/Makefile.am create mode 100644 starpupy/src/Makefile.am rename starpupy/src/starpu/{Makefile => Makefile.old} (85%) create mode 100644 starpupy/tests/Makefile.am rename starpupy/tests/{Makefile => Makefile.old} (100%) create mode 100755 starpupy/tests/execute.sh.in create mode 100755 starpupy/tests/starpu_py.sh create mode 100755 starpupy/tests/starpu_py_parallel.sh diff --git a/Makefile.am b/Makefile.am index 7a94252386..1bb3b11dee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -53,6 +53,10 @@ if STARPU_BUILD_STARPURM SUBDIRS += starpurm endif +if STARPU_BUILD_STARPUPY +SUBDIRS += starpupy +endif + if STARPU_BUILD_SC_HYPERVISOR SUBDIRS += sc_hypervisor endif diff --git a/configure.ac b/configure.ac index 738e6e4cbd..4e88ba7e1f 100644 --- a/configure.ac +++ b/configure.ac @@ -3394,6 +3394,40 @@ AC_ARG_ENABLE(starpurm-examples, [AS_HELP_STRING([--enable-starpurm-examples], enable_starpurm_examples=$enableval, enable_starpurm_examples=no) AM_CONDITIONAL(STARPU_BUILD_STARPURM_EXAMPLES, [test x$enable_starpurm_examples = xyes]) +##################################### +# StarPUPy # +##################################### + +starpupy_support=no +AC_ARG_ENABLE(starpupy, [AS_HELP_STRING([--enable-starpupy], [enable StarPU python interface])], + enable_starpupy=$enableval, enable_starpupy=no) +if test "x$enable_starpupy" != xno +then + starpupy_support=yes + AC_CHECK_PROGS([PYTHON_CONFIG], python3-config) + if test "$ac_cv_prog_PYTHON_CONFIG" == "" + then + AC_MSG_ERROR([python3-config missing, cannot build StarPU python interface]) + fi + AC_CHECK_PROGS([PYTHON], python3) + if test "$ac_cv_prog_PYTHON" == "" + then + AC_MSG_ERROR([python3 missing, cannot build StarPU python interface]) + fi + AC_SUBST(PYTHON) + SAVED_CPPFLAGS="${CPPFLAGS}" + CPPFLAGS="$CPPFLAGS $($ac_cv_prog_PYTHON_CONFIG --includes)" + AC_CHECK_HEADER([Python.h],[have_valid_starpupy=yes],[have_valid_starpupy=no]) + if test "$have_valid_starpupy" = "no" + then + AC_MSG_ERROR([Python.h missing, cannot build StarPU python interface]) + fi + CPPFLAGS="$SAVED_CPPFLAGS" + STARPUPY_CPPFLAGS="$($ac_cv_prog_PYTHON_CONFIG --includes)" + AC_SUBST(STARPUPY_CPPFLAGS) +fi +AM_CONDITIONAL(STARPU_BUILD_STARPUPY, [test x$starpupy_support = xyes]) + ########################################## # Documentation # ########################################## @@ -3555,6 +3589,10 @@ AC_CONFIG_COMMANDS([executable-scripts], [ test -e tools/starpu_paje_state_stats.R || ln -sf $ac_abs_top_srcdir/tools/starpu_paje_state_stats.R tools/starpu_paje_state_stats.R test -e tools/starpu_trace_state_stats.py || ln -sf $ac_abs_top_srcdir/tools/starpu_trace_state_stats.py tools/starpu_trace_state_stats.py chmod +x tools/starpu_trace_state_stats.py + chmod +x starpupy/tests/execute.sh + for x in starpupy/tests/starpu_py.sh starpupy/tests/starpu_py_parallel.sh; do + test -e $x || mkdir -p $(dirname $x) && ln -sf $ac_abs_top_srcdir/$x $(dirname $x) + done chmod +x julia/examples/execute.sh for x in julia/examples/check_deps/check_deps.sh julia/examples/mult/mult_starpu.sh julia/examples/mult/perf.sh julia/examples/variable/variable.sh julia/examples/task_insert_color/task_insert_color.sh julia/examples/vector_scal/vector_scal.sh julia/examples/mandelbrot/mandelbrot.sh julia/examples/callback/callback.sh julia/examples/dependency/task_dep.sh julia/examples/dependency/tag_dep.sh julia/examples/dependency/end_dep.sh julia/examples/axpy/axpy.sh julia/examples/gemm/gemm.sh julia/examples/cholesky/cholesky.sh; do test -e $x || mkdir -p $(dirname $x) && ln -sf $ac_abs_top_srcdir/$x $(dirname $x) @@ -3624,6 +3662,10 @@ AC_OUTPUT([ starpurm/src/Makefile starpurm/examples/Makefile starpurm/starpurm-1.3.pc + starpupy/Makefile + starpupy/src/Makefile + starpupy/tests/Makefile + starpupy/tests/execute.sh examples/Makefile examples/stencil/Makefile tests/Makefile @@ -3698,6 +3740,7 @@ AC_MSG_NOTICE([ Master-Slave MPI enabled: $use_mpi_master_slave FFT Support: $fft_support Resource Management enabled: $starpurm_support + Python Interface enabled: $starpupy_support OpenMP runtime support enabled: $enable_openmp Cluster support enabled: $enable_cluster SOCL enabled: $build_socl diff --git a/starpupy/Makefile.am b/starpupy/Makefile.am new file mode 100644 index 0000000000..9304706c7d --- /dev/null +++ b/starpupy/Makefile.am @@ -0,0 +1,30 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +include $(top_srcdir)/starpu-subdirtests.mk + +SUBDIRS = src +SUBDIRS += tests + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = starpupy-1.3.pc + +#versincludedir = $(includedir)/starpu/$(STARPU_EFFECTIVE_VERSION) +#versinclude_HEADERS = \ +# include/starpurm.h + +#nodist_versinclude_HEADERS = \ +# include/starpurm_config.h diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am new file mode 100644 index 0000000000..83ba45e31e --- /dev/null +++ b/starpupy/src/Makefile.am @@ -0,0 +1,61 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +include $(top_srcdir)/starpu-notests.mk + +SUBDIRS = + +#CLEANFILES = *.gcno *.gcda *.linkinfo + +AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include +AM_CFLAGS = -Wall -g $(STARPU_CUDA_CPPFLAGS) $(STARPU_OPENCL_CPPFLAGS) $(FXT_CFLAGS) $(MAGMA_CFLAGS) $(HWLOC_CFLAGS) $(GLOBAL_AM_CFLAGS) $(NMAD_CFLAGS) $(STARPUPY_CPPFLAGS) +AM_LDFLAGS = $(STARPU_OPENCL_LDFLAGS) $(STARPU_CUDA_LDFLAGS) $(STARPU_COI_LDFLAGS) $(STARPU_SCIF_LDFLAGS) $(NMAD_LDFLAGS) +#AM_LIBADD = + +PYTHON ?= python3 +AM_CPPFLAGS += $(shell $(PYTHON)-config --includes) -Wall -O2 -g + +all: starpu/starpupy.so + for f in $(top_srcdir)/starpupy/src/starpu/*py ; do cp $$f starpu ; done +starpu/starpupy.so: starpu/starpu_task_wrapper.c + $(MKDIR_P) starpu + $(CC) -fPIC $(AM_CFLAGS) $< -o $@ -shared $(AM_CPPFLAGS) $(AM_LDFLAGS) -L../../src/.libs -lstarpu-1.3 + +CLEANFILES= starpu/starpupy.so + +#libstarpupy_so_version = $(LIBSTARPUPY_INTERFACE_CURRENT):$(LIBSTARPUPY_INTERFACE_REVISION):$(LIBSTARPUPY_INTERFACE_AGE) +# +##lib_LTLIBRARIES = libstarpupy-@STARPU_EFFECTIVE_VERSION@.la +#lib_LTLIBRARIES = libstarpupy.la +# +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CPPFLAGS = $(AM_CPPFLAGS) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CFLAGS = $(AM_CFLAGS) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) +# +#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = \ +# starpu_task_wrapper.c +# +#libstarpupy_la_CPPFLAGS = $(AM_CPPFLAGS) +#libstarpupy_la_CFLAGS = $(AM_CFLAGS) +#libstarpupy_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) +#libstarpupy_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) +# +#libstarpupy_la_SOURCES = \ +# starpu_task_wrapper.c + +#noinst_HEADERS = \ +# starpurm_private.h diff --git a/starpupy/src/starpu/Makefile b/starpupy/src/starpu/Makefile.old similarity index 85% rename from starpupy/src/starpu/Makefile rename to starpupy/src/starpu/Makefile.old index a65543b86f..0080f7d827 100644 --- a/starpupy/src/starpu/Makefile +++ b/starpupy/src/starpu/Makefile.old @@ -6,7 +6,7 @@ LDLIBS += $(shell pkg-config --libs starpu-1.3) all: starpupy.so -starpupy.so: starpu_task_wrapper.c Makefile +starpupy.so: starpu_task_wrapper.c Makefile.old $(CC) -fPIC $(CFLAGS) $< -o $@ -shared $(CPPFLAGS) $(LDLIBS) clean: diff --git a/starpupy/tests/Makefile.am b/starpupy/tests/Makefile.am new file mode 100644 index 0000000000..105afb2509 --- /dev/null +++ b/starpupy/tests/Makefile.am @@ -0,0 +1,29 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# +SUBDIRS = + +CLEANFILES = *.gcno *.gcda *.linkinfo + +#AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src -I$(top_builddir)/src -I$(top_builddir)/include +#AM_CPPFLAGS += -I$(top_srcdir)/starpurm/include -I$(top_srcdir)/starpurm/src -I$(top_builddir)/starpurm/src -I$(top_builddir)/starpurm/include +#AM_CFLAGS = -Wall -g $(HWLOC_CFLAGS) $(DLB_CFLAGS) +#LDADD = $(top_builddir)/starpurm/src/libstarpurm-@STARPU_EFFECTIVE_VERSION@.la $(top_builddir)/src/@LIBSTARPU_LINK@ $(HWLOC_LIBS) $(DLB_LIBS) + +examplebindir = $(libdir)/starpupy/examples + +TESTS = +TESTS += starpu_py.sh +TESTS += starpu_py_parallel.sh diff --git a/starpupy/tests/Makefile b/starpupy/tests/Makefile.old similarity index 100% rename from starpupy/tests/Makefile rename to starpupy/tests/Makefile.old diff --git a/starpupy/tests/execute.sh.in b/starpupy/tests/execute.sh.in new file mode 100755 index 0000000000..01757c8cac --- /dev/null +++ b/starpupy/tests/execute.sh.in @@ -0,0 +1,34 @@ +#!@REALBASH@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +set -x + +tstdir=@STARPU_SRC_DIR@/starpupy/tests + +modpath=@STARPU_BUILD_DIR@/src/.libs: +pypath=@STARPU_BUILD_DIR@/starpupy/src:$PYTHONPATH + +tstfile=$1 +if test ! -f $tstdir/$tstfile +then + echo "Error. File $tstdir/$tstfile not found" + exit 1 +fi +shift + +PYTHONPATH=$pypath LD_LIBRARY_PATH=$modpath @PYTHON@ $tstdir/$tstfile $* + diff --git a/starpupy/tests/starpu_py.sh b/starpupy/tests/starpu_py.sh new file mode 100755 index 0000000000..df367755d3 --- /dev/null +++ b/starpupy/tests/starpu_py.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +$(dirname $0)/execute.sh starpu_py.py + diff --git a/starpupy/tests/starpu_py_parallel.sh b/starpupy/tests/starpu_py_parallel.sh new file mode 100755 index 0000000000..601543c38e --- /dev/null +++ b/starpupy/tests/starpu_py_parallel.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# + +$(dirname $0)/execute.sh starpu_py_parallel.py + -- GitLab From 14bf9085e0c5e00697e66300a7dcb280861a83b1 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Tue, 3 Nov 2020 11:06:19 +0100 Subject: [PATCH 03/16] starpupy: use distutils to create and install package --- configure.ac | 20 +++++------------ starpupy/Makefile.am | 9 -------- starpupy/src/Makefile.am | 46 +++++---------------------------------- starpupy/src/setup.cfg.in | 4 ++++ starpupy/src/setup.py.in | 33 ++++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 64 deletions(-) create mode 100644 starpupy/src/setup.cfg.in create mode 100644 starpupy/src/setup.py.in diff --git a/configure.ac b/configure.ac index 4e88ba7e1f..9a4356f9a1 100644 --- a/configure.ac +++ b/configure.ac @@ -56,6 +56,9 @@ AC_SUBST([LIBSTARPUFFT_INTERFACE_AGE]) AC_SUBST([LIBSTARPURM_INTERFACE_CURRENT]) AC_SUBST([LIBSTARPURM_INTERFACE_REVISION]) AC_SUBST([LIBSTARPURM_INTERFACE_AGE]) +AC_SUBST([LIBSTARPUPY_INTERFACE_CURRENT]) +AC_SUBST([LIBSTARPUPY_INTERFACE_REVISION]) +AC_SUBST([LIBSTARPUPY_INTERFACE_AGE]) AC_SUBST([LIBSOCL_INTERFACE_CURRENT]) AC_SUBST([LIBSOCL_INTERFACE_REVISION]) AC_SUBST([LIBSOCL_INTERFACE_AGE]) @@ -3404,27 +3407,12 @@ AC_ARG_ENABLE(starpupy, [AS_HELP_STRING([--enable-starpupy], [enable StarPU pyth if test "x$enable_starpupy" != xno then starpupy_support=yes - AC_CHECK_PROGS([PYTHON_CONFIG], python3-config) - if test "$ac_cv_prog_PYTHON_CONFIG" == "" - then - AC_MSG_ERROR([python3-config missing, cannot build StarPU python interface]) - fi AC_CHECK_PROGS([PYTHON], python3) if test "$ac_cv_prog_PYTHON" == "" then AC_MSG_ERROR([python3 missing, cannot build StarPU python interface]) fi AC_SUBST(PYTHON) - SAVED_CPPFLAGS="${CPPFLAGS}" - CPPFLAGS="$CPPFLAGS $($ac_cv_prog_PYTHON_CONFIG --includes)" - AC_CHECK_HEADER([Python.h],[have_valid_starpupy=yes],[have_valid_starpupy=no]) - if test "$have_valid_starpupy" = "no" - then - AC_MSG_ERROR([Python.h missing, cannot build StarPU python interface]) - fi - CPPFLAGS="$SAVED_CPPFLAGS" - STARPUPY_CPPFLAGS="$($ac_cv_prog_PYTHON_CONFIG --includes)" - AC_SUBST(STARPUPY_CPPFLAGS) fi AM_CONDITIONAL(STARPU_BUILD_STARPUPY, [test x$starpupy_support = xyes]) @@ -3662,6 +3650,8 @@ AC_OUTPUT([ starpurm/src/Makefile starpurm/examples/Makefile starpurm/starpurm-1.3.pc + starpupy/src/setup.cfg + starpupy/src/setup.py starpupy/Makefile starpupy/src/Makefile starpupy/tests/Makefile diff --git a/starpupy/Makefile.am b/starpupy/Makefile.am index 9304706c7d..ecf1f667a3 100644 --- a/starpupy/Makefile.am +++ b/starpupy/Makefile.am @@ -19,12 +19,3 @@ include $(top_srcdir)/starpu-subdirtests.mk SUBDIRS = src SUBDIRS += tests -pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = starpupy-1.3.pc - -#versincludedir = $(includedir)/starpu/$(STARPU_EFFECTIVE_VERSION) -#versinclude_HEADERS = \ -# include/starpurm.h - -#nodist_versinclude_HEADERS = \ -# include/starpurm_config.h diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am index 83ba45e31e..a12813a647 100644 --- a/starpupy/src/Makefile.am +++ b/starpupy/src/Makefile.am @@ -14,48 +14,14 @@ # See the GNU Lesser General Public License in COPYING.LGPL for more details. # -include $(top_srcdir)/starpu-notests.mk - SUBDIRS = -#CLEANFILES = *.gcno *.gcda *.linkinfo - -AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -AM_CFLAGS = -Wall -g $(STARPU_CUDA_CPPFLAGS) $(STARPU_OPENCL_CPPFLAGS) $(FXT_CFLAGS) $(MAGMA_CFLAGS) $(HWLOC_CFLAGS) $(GLOBAL_AM_CFLAGS) $(NMAD_CFLAGS) $(STARPUPY_CPPFLAGS) -AM_LDFLAGS = $(STARPU_OPENCL_LDFLAGS) $(STARPU_CUDA_LDFLAGS) $(STARPU_COI_LDFLAGS) $(STARPU_SCIF_LDFLAGS) $(NMAD_LDFLAGS) -#AM_LIBADD = - -PYTHON ?= python3 -AM_CPPFLAGS += $(shell $(PYTHON)-config --includes) -Wall -O2 -g - -all: starpu/starpupy.so - for f in $(top_srcdir)/starpupy/src/starpu/*py ; do cp $$f starpu ; done -starpu/starpupy.so: starpu/starpu_task_wrapper.c +all: $(MKDIR_P) starpu - $(CC) -fPIC $(AM_CFLAGS) $< -o $@ -shared $(AM_CPPFLAGS) $(AM_LDFLAGS) -L../../src/.libs -lstarpu-1.3 - -CLEANFILES= starpu/starpupy.so + for f in $(top_srcdir)/starpupy/src/starpu/*py ; do cp $$f starpu ; done + for f in $(top_srcdir)/starpupy/src/starpu/*c ; do cp $$f starpu ; done + $(PYTHON) setup.py build -#libstarpupy_so_version = $(LIBSTARPUPY_INTERFACE_CURRENT):$(LIBSTARPUPY_INTERFACE_REVISION):$(LIBSTARPUPY_INTERFACE_AGE) -# -##lib_LTLIBRARIES = libstarpupy-@STARPU_EFFECTIVE_VERSION@.la -#lib_LTLIBRARIES = libstarpupy.la -# -#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CPPFLAGS = $(AM_CPPFLAGS) -#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_CFLAGS = $(AM_CFLAGS) -#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) -#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) -# -#libstarpupy_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = \ -# starpu_task_wrapper.c -# -#libstarpupy_la_CPPFLAGS = $(AM_CPPFLAGS) -#libstarpupy_la_CFLAGS = $(AM_CFLAGS) -#libstarpupy_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info $(libstarpupy_so_version) -#libstarpupy_la_LIBADD = $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) -# -#libstarpupy_la_SOURCES = \ -# starpu_task_wrapper.c +install: + $(PYTHON) setup.py install -#noinst_HEADERS = \ -# starpurm_private.h diff --git a/starpupy/src/setup.cfg.in b/starpupy/src/setup.cfg.in new file mode 100644 index 0000000000..c98060bd82 --- /dev/null +++ b/starpupy/src/setup.cfg.in @@ -0,0 +1,4 @@ +[install] +prefix=@prefix@ + + diff --git a/starpupy/src/setup.py.in b/starpupy/src/setup.py.in new file mode 100644 index 0000000000..c1e4bdd04a --- /dev/null +++ b/starpupy/src/setup.py.in @@ -0,0 +1,33 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# +from distutils.core import setup, Extension + +starpupy = Extension('starpupy', + include_dirs = ['@STARPU_SRC_DIR@/include', '@STARPU_BUILD_DIR@/include'], + sources = ['starpu/starpu_task_wrapper.c']) + +setup( + name = 'starpupy', + version = '0.5', + description = 'Python bindings for StarPU', + author = 'StarPU team', + author_email = 'starpu-devel@lists.gforge.inria.fr', + url = 'https://starpu.gitlabpages.inria.fr/', + license = 'GPL', + platforms = 'posix', + ext_modules = [starpupy], + packages = ['starpu'], + ) -- GitLab From 9867a60628700cacc7bdfba45504d59f9a2dd6e9 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Tue, 3 Nov 2020 11:11:07 +0100 Subject: [PATCH 04/16] starpupy: add copyright --- starpupy/src/setup.cfg.in | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/starpupy/src/setup.cfg.in b/starpupy/src/setup.cfg.in index c98060bd82..d0b6a85f2a 100644 --- a/starpupy/src/setup.cfg.in +++ b/starpupy/src/setup.cfg.in @@ -1,3 +1,18 @@ +# StarPU --- Runtime system for heterogeneous multicore architectures. +# +# Copyright (C) 2020 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria +# +# StarPU is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or (at +# your option) any later version. +# +# StarPU is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Lesser General Public License in COPYING.LGPL for more details. +# [install] prefix=@prefix@ -- GitLab From 3652d29321775f7b703ae13777cba296c8185fe6 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Tue, 3 Nov 2020 11:34:23 +0100 Subject: [PATCH 05/16] starpupy: fix for running tests --- starpupy/src/setup.py.in | 4 +++- starpupy/tests/Makefile.am | 5 ----- starpupy/tests/execute.sh.in | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/starpupy/src/setup.py.in b/starpupy/src/setup.py.in index c1e4bdd04a..95144ab1fc 100644 --- a/starpupy/src/setup.py.in +++ b/starpupy/src/setup.py.in @@ -15,8 +15,10 @@ # from distutils.core import setup, Extension -starpupy = Extension('starpupy', +starpupy = Extension('starpu.starpupy', include_dirs = ['@STARPU_SRC_DIR@/include', '@STARPU_BUILD_DIR@/include'], + libraries = ['starpu-@STARPU_EFFECTIVE_VERSION@'], + library_dirs = ['@STARPU_BUILD_DIR@/src/.libs'], sources = ['starpu/starpu_task_wrapper.c']) setup( diff --git a/starpupy/tests/Makefile.am b/starpupy/tests/Makefile.am index 105afb2509..4dd4700ca9 100644 --- a/starpupy/tests/Makefile.am +++ b/starpupy/tests/Makefile.am @@ -17,11 +17,6 @@ SUBDIRS = CLEANFILES = *.gcno *.gcda *.linkinfo -#AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src -I$(top_builddir)/src -I$(top_builddir)/include -#AM_CPPFLAGS += -I$(top_srcdir)/starpurm/include -I$(top_srcdir)/starpurm/src -I$(top_builddir)/starpurm/src -I$(top_builddir)/starpurm/include -#AM_CFLAGS = -Wall -g $(HWLOC_CFLAGS) $(DLB_CFLAGS) -#LDADD = $(top_builddir)/starpurm/src/libstarpurm-@STARPU_EFFECTIVE_VERSION@.la $(top_builddir)/src/@LIBSTARPU_LINK@ $(HWLOC_LIBS) $(DLB_LIBS) - examplebindir = $(libdir)/starpupy/examples TESTS = diff --git a/starpupy/tests/execute.sh.in b/starpupy/tests/execute.sh.in index 01757c8cac..87facc83f2 100755 --- a/starpupy/tests/execute.sh.in +++ b/starpupy/tests/execute.sh.in @@ -20,7 +20,7 @@ set -x tstdir=@STARPU_SRC_DIR@/starpupy/tests modpath=@STARPU_BUILD_DIR@/src/.libs: -pypath=@STARPU_BUILD_DIR@/starpupy/src:$PYTHONPATH +pypath=@STARPU_BUILD_DIR@/starpupy/src/build/lib.linux-x86_64-3.8/:$PYTHONPATH tstfile=$1 if test ! -f $tstdir/$tstfile -- GitLab From 194154f77cd6e8dc704bd1e477feeb12c268bfc1 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Tue, 3 Nov 2020 11:46:26 +0100 Subject: [PATCH 06/16] starpupy: minor fixes --- starpupy/src/Makefile.am | 6 ++++++ starpupy/src/setup.cfg.in | 4 ++++ starpupy/tests/execute.sh.in | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am index a12813a647..c559e8f8f5 100644 --- a/starpupy/src/Makefile.am +++ b/starpupy/src/Makefile.am @@ -25,3 +25,9 @@ all: install: $(PYTHON) setup.py install +clean: + $(PYTHON) setup.py clean + +distclean: clean + rm -rf build + diff --git a/starpupy/src/setup.cfg.in b/starpupy/src/setup.cfg.in index d0b6a85f2a..297c337502 100644 --- a/starpupy/src/setup.cfg.in +++ b/starpupy/src/setup.cfg.in @@ -13,6 +13,10 @@ # # See the GNU Lesser General Public License in COPYING.LGPL for more details. # +[build] +build-platlib=build +build-temp=build/tmp + [install] prefix=@prefix@ diff --git a/starpupy/tests/execute.sh.in b/starpupy/tests/execute.sh.in index 87facc83f2..308b4146d9 100755 --- a/starpupy/tests/execute.sh.in +++ b/starpupy/tests/execute.sh.in @@ -20,7 +20,7 @@ set -x tstdir=@STARPU_SRC_DIR@/starpupy/tests modpath=@STARPU_BUILD_DIR@/src/.libs: -pypath=@STARPU_BUILD_DIR@/starpupy/src/build/lib.linux-x86_64-3.8/:$PYTHONPATH +pypath=@STARPU_BUILD_DIR@/starpupy/src/build:$PYTHONPATH tstfile=$1 if test ! -f $tstdir/$tstfile -- GitLab From 7b0512a2c0aa048d678995546e71368cefcf8a39 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Tue, 3 Nov 2020 18:58:20 +0100 Subject: [PATCH 07/16] starpupy: add files to dist target --- configure.ac | 3 --- starpupy/src/Makefile.am | 14 ++++++++++++-- starpupy/tests/Makefile.am | 6 ++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 9a4356f9a1..515e69ed55 100644 --- a/configure.ac +++ b/configure.ac @@ -56,9 +56,6 @@ AC_SUBST([LIBSTARPUFFT_INTERFACE_AGE]) AC_SUBST([LIBSTARPURM_INTERFACE_CURRENT]) AC_SUBST([LIBSTARPURM_INTERFACE_REVISION]) AC_SUBST([LIBSTARPURM_INTERFACE_AGE]) -AC_SUBST([LIBSTARPUPY_INTERFACE_CURRENT]) -AC_SUBST([LIBSTARPUPY_INTERFACE_REVISION]) -AC_SUBST([LIBSTARPUPY_INTERFACE_AGE]) AC_SUBST([LIBSOCL_INTERFACE_CURRENT]) AC_SUBST([LIBSOCL_INTERFACE_REVISION]) AC_SUBST([LIBSOCL_INTERFACE_AGE]) diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am index c559e8f8f5..13380b9b4d 100644 --- a/starpupy/src/Makefile.am +++ b/starpupy/src/Makefile.am @@ -25,9 +25,19 @@ all: install: $(PYTHON) setup.py install -clean: +clean-local: $(PYTHON) setup.py clean -distclean: clean +distclean-local: clean rm -rf build +uninstall: + rm -rf $(prefix)/lib/python*/site-packages + +EXTRA_DIST = \ + starpu/delay.py \ + starpu/__init__.py \ + starpu/intermedia.py \ + starpu/joblib.py \ + starpu/starpu_task_wrapper.c + diff --git a/starpupy/tests/Makefile.am b/starpupy/tests/Makefile.am index 4dd4700ca9..a70a8a43ee 100644 --- a/starpupy/tests/Makefile.am +++ b/starpupy/tests/Makefile.am @@ -22,3 +22,9 @@ examplebindir = $(libdir)/starpupy/examples TESTS = TESTS += starpu_py.sh TESTS += starpu_py_parallel.sh + +EXTRA_DIST = \ + starpu_py_parallel.py \ + starpu_py_parallel.sh \ + starpu_py.py \ + starpu_py.sh -- GitLab From cf2b0f3335937b9539b2fe7a8ce4b8218293c775 Mon Sep 17 00:00:00 2001 From: HE Kun Date: Wed, 4 Nov 2020 17:26:59 +0100 Subject: [PATCH 08/16] starpupy: change the function parallel to class Parallel --- doc/doxygen/chapters/400_python.doxy | 25 +++++--- starpupy/src/starpu/joblib.py | 90 +++++++++++++++++++++------- starpupy/tests/starpu_py_parallel.py | 12 ++-- 3 files changed, 93 insertions(+), 34 deletions(-) diff --git a/doc/doxygen/chapters/400_python.doxy b/doc/doxygen/chapters/400_python.doxy index 328221f7ee..7ac7585af9 100644 --- a/doc/doxygen/chapters/400_python.doxy +++ b/doc/doxygen/chapters/400_python.doxy @@ -189,11 +189,20 @@ StarPU Python interface also provides parallel computing for loops using multipr [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] \endcode -In order to spread it over several CPUs, you can use starpu.joblib module, and call parallel function: +In order to spread it over several CPUs, you can use starpu.joblib module, and call Parallel class: \code{.py} >>> from math import log10 ->>> starpu.joblib.parallel(mode="normal", n_jobs=2)(starpu.joblib.delayed(log10)(10**i)for i in range(10)) +>>> starpu.joblib.Parallel(mode="normal", n_jobs=2)(starpu.joblib.delayed(log10)(10**i)for i in range(10)) +[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] +\endcode + +Or you can create an object of Parallel class, and then call starpu.joblib.delayed to execute the function. + +\code{.py} +>>> from math import log10 +>>> parallel=starpu.joblib.Parallel(mode="normal", n_jobs=2) +>>> parallel(starpu.joblib.delayed(log10)(10**i)for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] \endcode @@ -231,7 +240,7 @@ def sub(a, b, c): g_func.append(starpu.joblib.delayed(sub)(6, 2, 5.9)) #input is iterable function list -starpu.joblib.parallel(mode="normal", n_jobs=2)(g_func) +starpu.joblib.Parallel(mode="normal", n_jobs=2)(g_func) \endcode Execution: @@ -245,18 +254,18 @@ Example 4: The result of 6 - 2 - 5.9 is -1.9000000000000004 and the result of 6 \subsection ParallelParameters Parallel Parameters -Without setting options of function task_submit, joblib.parallel module also provides some own parameters: +Without setting options of function task_submit, starpu.joblib.Parallel also provides some own parameters: \subsubsection mode mode (string, default: "normal") -You need to choose the mode between normal and future. As in the previous example, with normal mode, you can call starpu.joblib.parallel directly without using asyncio module and you will get the result when the task is executed. With future mode, when you call starpu.joblib.parallel, you will get a Future object as return value. Here if you set another parameter end_msg, you will receive a signal with this message that the result is ready, then you can perform awaiting to get the eventual result. The asyncio module should be imported in this case. +You need to choose the mode between normal and future. As in the previous example, with normal mode, you can call starpu.joblib.Parallel directly without using asyncio module and you will get the result when the task is executed. With future mode, when you call starpu.joblib.Parallel, you will get a Future object as return value. Here if you set another parameter end_msg, you will receive a signal with this message that the result is ready, then you can perform awaiting to get the eventual result. The asyncio module should be imported in this case. \code{.py} >>> import starpu >>> import asyncio >>> from math import log10 ->>> fut = starpu.joblib.parallel(mode="future", n_jobs=3, end_msg="The result is ready!")(starpu.joblib.delayed(log10)(10**i)for i in range(10)) +>>> fut = starpu.joblib.Parallel(mode="future", n_jobs=3, end_msg="The result is ready!")(starpu.joblib.delayed(log10)(10**i)for i in range(10)) >>> The result is ready! <_GatheringFuture finished result=[[0.0, 1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]> >>> await fut [[0.0, 1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] @@ -269,7 +278,7 @@ You need to choose the mode between normal and future. As in the p As we introduced in the previous section, this parameter can be set with a prompt message to remind you that the task is executed and the result is ready, then you can perform awaiting and get the eventual result. If you do not set this parameter, the default value is None, and you will not receive any prompt message, but you still can perform awaiting and get the eventual result. \subsubsection n_jobs n_jobs -(int, default: 1) +(int, default: None) You need to set the number of CPUs which is used for parallel computing. Thus for n_jobs=2, 2 CPUs are used. If 1 is given, no parallel computing. For n_jobs below 0, (n_cpus+1+n_jobs) CPUs are used. Thus for n_jobs=-2, all CPUs but one are used. @@ -279,7 +288,7 @@ In the following example, for the function log10 (i+1) for i in range(N), we set >>> from math import log10 >>> for x in [10, 100, 1000, 10000, 100000, 1000000]: ... for N in range(x, x*10, x): -... starpu.joblib.parallel(mode="normal", n_jobs=2, perfmodel="log")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) +... starpu.joblib.Parallel(mode="normal", n_jobs=2, perfmodel="log")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) \endcode You can call the function perfmodel_plot by giving the symbol of perfmodel to view the performance curve. diff --git a/starpupy/src/starpu/joblib.py b/starpupy/src/starpu/joblib.py index c8935a82ca..ba72d268d8 100644 --- a/starpupy/src/starpu/joblib.py +++ b/starpupy/src/starpu/joblib.py @@ -15,12 +15,15 @@ # from starpu import starpupy import starpu +import joblib import asyncio import math import functools # get the number of CPUs controlled by StarPU -n_cpus=starpupy.cpu_worker_get_count() +def cpu_count(): + n_cpus=starpupy.cpu_worker_get_count() + return n_cpus # split a list ls into n_block numbers of sub-lists def partition(ls, n_block): @@ -55,10 +58,10 @@ def future_generator(g, n_jobs, dict_task): L_func.append(f(*L_args)) return L_func # get the number of block - if n_jobs<-n_cpus-1 or n_jobs>n_cpus: - print("Error: n_jobs is out of range, number of CPUs is", n_cpus) + if n_jobs<-cpu_count()-1 or n_jobs>cpu_count(): + print("Error: n_jobs is out of range, number of CPUs is", cpu_count()) elif n_jobs<0: - n_block=n_cpus+1+n_jobs + n_block=cpu_count()+1+n_jobs else: n_block=n_jobs # generate the split function list @@ -72,18 +75,34 @@ def future_generator(g, n_jobs, dict_task): L_fut.append(fut) return L_fut -def parallel(*, mode="normal", n_jobs=1, perfmodel=None, end_msg=None,\ +class Parallel(joblib.Parallel): + def __init__(self, mode="normal", perfmodel=None, end_msg=None,\ name=None, synchronous=0, priority=0, color=None, flops=None,\ - backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ + n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ batch_size='auto', temp_folder=None, max_nbytes='1M',\ mmap_mode='r', prefer=None, require=None): - #generate the dictionary of task_submit - dict_task={'name': name, 'synchronous': synchronous, 'priority': priority, 'color': color, 'flops': flops, 'perfmodel': perfmodel} - # the mode normal, user can call the function directly without using async - if mode=="normal": - def parallel_normal(g): + super(Parallel, self).__init__(n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ + batch_size='auto', temp_folder=None, max_nbytes='1M',\ + mmap_mode='r', prefer=None, require=None) + + self.mode=mode + self.perfmodel=perfmodel + self.end_msg=end_msg + self.name=name + self.synchronous=synchronous + self.priority=priority + self.color=color + self.flops=flops + self.n_jobs=n_jobs + + def __call__(self,iterable): + #generate the dictionary of task_submit + dict_task={'name': self.name, 'synchronous': self.synchronous, 'priority': self.priority, 'color': self.color, 'flops': self.flops, 'perfmodel': self.perfmodel} + # the mode normal, user can call the function directly without using async + if self.mode=="normal": + #def parallel_normal(g): async def asy_main(): - L_fut=future_generator(g, n_jobs, dict_task) + L_fut=future_generator(iterable, self.n_jobs, dict_task) res=[] for i in range(len(L_fut)): L_res=await L_fut[i] @@ -92,19 +111,19 @@ def parallel(*, mode="normal", n_jobs=1, perfmodel=None, end_msg=None,\ return res asyncio.run(asy_main()) return asy_main - return parallel_normal - # the mode future, user needs to use asyncio module and await the Future result in main function - elif mode=="future": - def parallel_future(g): - L_fut=future_generator(g, n_jobs, dict_task) + #return parallel_normal + # the mode future, user needs to use asyncio module and await the Future result in main function + elif self.mode=="future": + #def parallel_future(g): + L_fut=future_generator(iterable, self.n_jobs, dict_task) fut=asyncio.gather(*L_fut) - if end_msg==None: + if self.end_msg==None: return fut else: - fut.add_done_callback(functools.partial(print, end_msg)) + fut.add_done_callback(functools.partial(print, self.end_msg)) return fut #return fut - return parallel_future + #return parallel_future def delayed(f): def delayed_func(*args): @@ -113,4 +132,35 @@ def delayed(f): ###################################################################### +class Memory(joblib.Memory): + def __init__(self,location=None, backend='local', cachedir=None, + mmap_mode=None, compress=False, verbose=1, bytes_limit=None, + backend_options=None): + super(Memory, self).__init__(location=None, backend='local', cachedir=None, + mmap_mode=None, compress=False, verbose=1, bytes_limit=None, + backend_options=None) + + +def dump(value, filename, compress=0, protocol=None, cache_size=None): + return joblib.dump(value, filename, compress, protocol, cache_size) + +def load(filename, mmap_mode=None): + return joblib.load(filename, mmap_mode) + +def hash(obj, hash_name='md5', coerce_mmap=False): + return joblib.hash(obj, hash_name, coerce_mmap) + +def register_compressor(compressor_name, compressor, force=False): + return joblib.register_compressor(compressor_name, compressor, force) + +def effective_n_jobs(n_jobs=-1): + return joblib.effective_n_jobs(n_jobs) + +class parallel_backend(joblib.parallel_backend): + def __init__(self, backend, n_jobs=-1, inner_max_num_threads=None, + **backend_params): + super(parallel_backend, self).__init__(backend, n_jobs=-1, inner_max_num_threads=None, + **backend_params) +def register_parallel_backend(name, factory, make_default=False): + return joblib.register_parallel_backend(name, factory, make_default) \ No newline at end of file diff --git a/starpupy/tests/starpu_py_parallel.py b/starpupy/tests/starpu_py_parallel.py index 31fae8515b..eeabca6cee 100644 --- a/starpupy/tests/starpu_py_parallel.py +++ b/starpupy/tests/starpu_py_parallel.py @@ -67,13 +67,13 @@ print("************************") print("parallel Normal version:") print("************************") print("--input is iterable argument list, example 1") -starpu.joblib.parallel(mode="normal", n_jobs=-2, perfmodel="first")(starpu.joblib.delayed(sqrt)(i**2)for i in range(N)) +starpu.joblib.Parallel(mode="normal", n_jobs=-2, perfmodel="first")(starpu.joblib.delayed(sqrt)(i**2)for i in range(N)) print("--input is iterable argument list, example 2") -starpu.joblib.parallel(mode="normal", n_jobs=2, perfmodel="second")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) +starpu.joblib.Parallel(mode="normal", n_jobs=2, perfmodel="second")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) print("--input is iterable function list") -starpu.joblib.parallel(mode="normal", n_jobs=3, perfmodel="third")(g_func) +starpu.joblib.Parallel(mode="normal", n_jobs=3, perfmodel="third")(g_func) print("************************") @@ -81,17 +81,17 @@ print("parallel Future version:") print("************************") async def main(): print("--input is iterable argument list, example 1") - fut1=starpu.joblib.parallel(mode="future", n_jobs=-3, perfmodel="first")(starpu.joblib.delayed(sqrt)(i**2)for i in range(N)) + fut1=starpu.joblib.Parallel(mode="future", n_jobs=-3, perfmodel="first")(starpu.joblib.delayed(sqrt)(i**2)for i in range(N)) res1=await fut1 #print(res1) print("--input is iterable argument list, example 2") - fut2=starpu.joblib.parallel(mode="future", n_jobs=-3, perfmodel="second")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) + fut2=starpu.joblib.Parallel(mode="future", n_jobs=-3, perfmodel="second")(starpu.joblib.delayed(log10)(i+1)for i in range(N)) res2=await fut2 #print(res2) print("--input is iterable function list") - fut3=starpu.joblib.parallel(mode="future", n_jobs=2, perfmodel="third")(g_func) + fut3=starpu.joblib.Parallel(mode="future", n_jobs=2, perfmodel="third")(g_func) res3=await fut3 #print(res3) asyncio.run(main()) -- GitLab From f2bd2f317d4f354270262924bd88643b36e55db6 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Wed, 4 Nov 2020 20:31:49 +0100 Subject: [PATCH 09/16] starpupy: do not show plot by default --- starpupy/src/starpu/intermedia.py | 9 +++++---- starpupy/tests/starpu_py_parallel.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/starpupy/src/starpu/intermedia.py b/starpupy/src/starpu/intermedia.py index bf137d2427..82c18848e5 100644 --- a/starpupy/src/starpu/intermedia.py +++ b/starpupy/src/starpu/intermedia.py @@ -54,9 +54,10 @@ def task_submit(*, name=None, synchronous=0, priority=0, color=None, flops=None, return call_task_submit # dump performance model and show the plot -def perfmodel_plot(perfmodel): +def perfmodel_plot(perfmodel, view=False): p=dict_perf[perfmodel] starpupy.save_history_based_model(p.get_struct()) - os.system('starpu_perfmodel_plot -s "' + perfmodel +'"') - os.system('gnuplot starpu_'+perfmodel+'.gp') - os.system('gv starpu_'+perfmodel+'.eps') \ No newline at end of file + if view == True: + os.system('starpu_perfmodel_plot -s "' + perfmodel +'"') + os.system('gnuplot starpu_'+perfmodel+'.gp') + os.system('gv starpu_'+perfmodel+'.eps') diff --git a/starpupy/tests/starpu_py_parallel.py b/starpupy/tests/starpu_py_parallel.py index eeabca6cee..88ed096a74 100644 --- a/starpupy/tests/starpu_py_parallel.py +++ b/starpupy/tests/starpu_py_parallel.py @@ -96,6 +96,6 @@ async def main(): #print(res3) asyncio.run(main()) -starpu.perfmodel_plot(perfmodel="first") +starpu.perfmodel_plot(perfmodel="first")#,view=True starpu.perfmodel_plot(perfmodel="second") -starpu.perfmodel_plot(perfmodel="third") \ No newline at end of file +starpu.perfmodel_plot(perfmodel="third") -- GitLab From 33daa1dea0b085dc5540a53a783b11d5b9f422fe Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Wed, 4 Nov 2020 20:23:09 +0100 Subject: [PATCH 10/16] starpupy/src/Makefile.am: fix rules --- starpupy/src/Makefile.am | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am index 13380b9b4d..9b52d8ab75 100644 --- a/starpupy/src/Makefile.am +++ b/starpupy/src/Makefile.am @@ -23,16 +23,22 @@ all: $(PYTHON) setup.py build install: + @if test -d $(prefix)/lib/python* ; \ + then \ + chmod -R u+w $(prefix)/lib/python* ; \ + fi $(PYTHON) setup.py install clean-local: $(PYTHON) setup.py clean + rm -f starpu/*py starpu/*c -distclean-local: clean +distclean-local: rm -rf build uninstall: - rm -rf $(prefix)/lib/python*/site-packages + rm -rf $(prefix)/lib/python*/site-packages/starpu* + rm -rf $(prefix)/lib/python*/site-packages/tmp/starpu* EXTRA_DIST = \ starpu/delay.py \ -- GitLab From bb41b1f8b454150d8e456ed5985ef89ddd4d6785 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Wed, 4 Nov 2020 20:34:27 +0100 Subject: [PATCH 11/16] starpupy: enable by default in branch python --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 515e69ed55..9fbcffcfd7 100644 --- a/configure.ac +++ b/configure.ac @@ -3400,7 +3400,7 @@ AM_CONDITIONAL(STARPU_BUILD_STARPURM_EXAMPLES, [test x$enable_starpurm_examples starpupy_support=no AC_ARG_ENABLE(starpupy, [AS_HELP_STRING([--enable-starpupy], [enable StarPU python interface])], - enable_starpupy=$enableval, enable_starpupy=no) + enable_starpupy=$enableval, enable_starpupy=yes) if test "x$enable_starpupy" != xno then starpupy_support=yes -- GitLab From 7fb9e9af99f7692a903e12674d28adc25cb32eb2 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Thu, 5 Nov 2020 07:58:47 +0100 Subject: [PATCH 12/16] starpupy: show plot by default --- starpupy/src/starpu/intermedia.py | 2 +- starpupy/tests/starpu_py_parallel.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/starpupy/src/starpu/intermedia.py b/starpupy/src/starpu/intermedia.py index 82c18848e5..4284f71fc4 100644 --- a/starpupy/src/starpu/intermedia.py +++ b/starpupy/src/starpu/intermedia.py @@ -54,7 +54,7 @@ def task_submit(*, name=None, synchronous=0, priority=0, color=None, flops=None, return call_task_submit # dump performance model and show the plot -def perfmodel_plot(perfmodel, view=False): +def perfmodel_plot(perfmodel, view=True): p=dict_perf[perfmodel] starpupy.save_history_based_model(p.get_struct()) if view == True: diff --git a/starpupy/tests/starpu_py_parallel.py b/starpupy/tests/starpu_py_parallel.py index 88ed096a74..99c2b4eaf1 100644 --- a/starpupy/tests/starpu_py_parallel.py +++ b/starpupy/tests/starpu_py_parallel.py @@ -96,6 +96,6 @@ async def main(): #print(res3) asyncio.run(main()) -starpu.perfmodel_plot(perfmodel="first")#,view=True -starpu.perfmodel_plot(perfmodel="second") -starpu.perfmodel_plot(perfmodel="third") +starpu.perfmodel_plot(perfmodel="first",view=False) +starpu.perfmodel_plot(perfmodel="second",view=False) +starpu.perfmodel_plot(perfmodel="third",view=False) -- GitLab From d4bdad29ce0762064fad35aa1eae548deaddf4fb Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Thu, 5 Nov 2020 08:06:03 +0100 Subject: [PATCH 13/16] starpupu/src/Makefile.am: no need to do a chmod -R --- starpupy/src/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/starpupy/src/Makefile.am b/starpupy/src/Makefile.am index 9b52d8ab75..47212e5cca 100644 --- a/starpupy/src/Makefile.am +++ b/starpupy/src/Makefile.am @@ -25,7 +25,8 @@ all: install: @if test -d $(prefix)/lib/python* ; \ then \ - chmod -R u+w $(prefix)/lib/python* ; \ + chmod u+w $(prefix)/lib/python* ; \ + chmod u+w $(prefix)/lib/python*/site-packages ; \ fi $(PYTHON) setup.py install -- GitLab From 86cb2a05a5fb6bc30670d09284922e5751b3fc0c Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Thu, 5 Nov 2020 08:13:07 +0100 Subject: [PATCH 14/16] starpupy/src/starpu/joblib.py: do not import joblib --- starpupy/src/starpu/joblib.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/starpupy/src/starpu/joblib.py b/starpupy/src/starpu/joblib.py index ba72d268d8..0276a3606b 100644 --- a/starpupy/src/starpu/joblib.py +++ b/starpupy/src/starpu/joblib.py @@ -15,7 +15,7 @@ # from starpu import starpupy import starpu -import joblib +#import joblib import asyncio import math import functools @@ -75,15 +75,15 @@ def future_generator(g, n_jobs, dict_task): L_fut.append(fut) return L_fut -class Parallel(joblib.Parallel): +class Parallel():#joblib.Parallel): def __init__(self, mode="normal", perfmodel=None, end_msg=None,\ name=None, synchronous=0, priority=0, color=None, flops=None,\ n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ batch_size='auto', temp_folder=None, max_nbytes='1M',\ mmap_mode='r', prefer=None, require=None): - super(Parallel, self).__init__(n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ - batch_size='auto', temp_folder=None, max_nbytes='1M',\ - mmap_mode='r', prefer=None, require=None) + #super(Parallel, self).__init__(n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ + #batch_size='auto', temp_folder=None, max_nbytes='1M',\ + #mmap_mode='r', prefer=None, require=None) self.mode=mode self.perfmodel=perfmodel @@ -132,7 +132,7 @@ def delayed(f): ###################################################################### -class Memory(joblib.Memory): +class Memory():#joblib.Memory): def __init__(self,location=None, backend='local', cachedir=None, mmap_mode=None, compress=False, verbose=1, bytes_limit=None, backend_options=None): @@ -156,11 +156,11 @@ def register_compressor(compressor_name, compressor, force=False): def effective_n_jobs(n_jobs=-1): return joblib.effective_n_jobs(n_jobs) -class parallel_backend(joblib.parallel_backend): +class parallel_backend():#joblib.parallel_backend): def __init__(self, backend, n_jobs=-1, inner_max_num_threads=None, **backend_params): super(parallel_backend, self).__init__(backend, n_jobs=-1, inner_max_num_threads=None, **backend_params) def register_parallel_backend(name, factory, make_default=False): - return joblib.register_parallel_backend(name, factory, make_default) \ No newline at end of file + return joblib.register_parallel_backend(name, factory, make_default) -- GitLab From 13a2ea579aefddc61217da13d6a2026d301171e5 Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Thu, 5 Nov 2020 09:45:20 +0100 Subject: [PATCH 15/16] Revert "starpupy/src/starpu/joblib.py: do not import joblib" This reverts commit 86cb2a05a5fb6bc30670d09284922e5751b3fc0c. --- starpupy/src/starpu/joblib.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/starpupy/src/starpu/joblib.py b/starpupy/src/starpu/joblib.py index 0276a3606b..ba72d268d8 100644 --- a/starpupy/src/starpu/joblib.py +++ b/starpupy/src/starpu/joblib.py @@ -15,7 +15,7 @@ # from starpu import starpupy import starpu -#import joblib +import joblib import asyncio import math import functools @@ -75,15 +75,15 @@ def future_generator(g, n_jobs, dict_task): L_fut.append(fut) return L_fut -class Parallel():#joblib.Parallel): +class Parallel(joblib.Parallel): def __init__(self, mode="normal", perfmodel=None, end_msg=None,\ name=None, synchronous=0, priority=0, color=None, flops=None,\ n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ batch_size='auto', temp_folder=None, max_nbytes='1M',\ mmap_mode='r', prefer=None, require=None): - #super(Parallel, self).__init__(n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ - #batch_size='auto', temp_folder=None, max_nbytes='1M',\ - #mmap_mode='r', prefer=None, require=None) + super(Parallel, self).__init__(n_jobs=None, backend=None, verbose=0, timeout=None, pre_dispatch='2 * n_jobs',\ + batch_size='auto', temp_folder=None, max_nbytes='1M',\ + mmap_mode='r', prefer=None, require=None) self.mode=mode self.perfmodel=perfmodel @@ -132,7 +132,7 @@ def delayed(f): ###################################################################### -class Memory():#joblib.Memory): +class Memory(joblib.Memory): def __init__(self,location=None, backend='local', cachedir=None, mmap_mode=None, compress=False, verbose=1, bytes_limit=None, backend_options=None): @@ -156,11 +156,11 @@ def register_compressor(compressor_name, compressor, force=False): def effective_n_jobs(n_jobs=-1): return joblib.effective_n_jobs(n_jobs) -class parallel_backend():#joblib.parallel_backend): +class parallel_backend(joblib.parallel_backend): def __init__(self, backend, n_jobs=-1, inner_max_num_threads=None, **backend_params): super(parallel_backend, self).__init__(backend, n_jobs=-1, inner_max_num_threads=None, **backend_params) def register_parallel_backend(name, factory, make_default=False): - return joblib.register_parallel_backend(name, factory, make_default) + return joblib.register_parallel_backend(name, factory, make_default) \ No newline at end of file -- GitLab From 0f352d0847533d35f747691c0188334c36d49e6d Mon Sep 17 00:00:00 2001 From: Nathalie Furmento Date: Thu, 5 Nov 2020 10:12:21 +0100 Subject: [PATCH 16/16] starpupy: check for python3 module joblib availability --- configure.ac | 6 ++++++ m4/libs.m4 | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/configure.ac b/configure.ac index 9fbcffcfd7..92dd56294e 100644 --- a/configure.ac +++ b/configure.ac @@ -3410,6 +3410,12 @@ then AC_MSG_ERROR([python3 missing, cannot build StarPU python interface]) fi AC_SUBST(PYTHON) + AC_MSG_CHECKING(for python3 module joblib) + AC_PYTHON_MODULE(joblib,[joblib_avail=yes],[joblib_avail=no]) + AC_MSG_RESULT($joblib_avail) + if test "$joblib_avail" = "no" ; then + AC_MSG_ERROR([python3 module joblib missing, cannot build StarPU python interface]) + fi fi AM_CONDITIONAL(STARPU_BUILD_STARPUPY, [test x$starpupy_support = xyes]) diff --git a/m4/libs.m4 b/m4/libs.m4 index 63b061afbf..19197d3552 100644 --- a/m4/libs.m4 +++ b/m4/libs.m4 @@ -71,3 +71,15 @@ AC_RUN_IFELSE([AC_LANG_PROGRAM( AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) ])dnl + +# AC_PYTHON_MODULE(modulename, [action-if-found], [action-if-not-found]) +# Check if the given python module is available +AC_DEFUN([AC_PYTHON_MODULE], + [ + echo "import $1" | $PYTHON - 2>/dev/null + if test $? -ne 0 ; then + $3 + else + $2 + fi +]) -- GitLab