Add environment variables to define a disk swap

doc/doxygen/chapters/15out_of_core.doxy

@@ -31,15 +31,23 @@ data handle. StarPU will then automatically read and write data as appropriate.
 To use a disk memory node, you have to register it with this function:
 
 \code{.c}
-	int new_dd = starpu_disk_register(&starpu_disk_stdio_ops, (void *) "/tmp/", 1024*1024*200);
+	int new_dd = starpu_disk_register(&starpu_disk_unistd_ops, (void *) "/tmp/", 1024*1024*200);
 \endcode
 
-Here, we use the stdio library to realize the read/write operations, i.e.
+Here, we use the unistd library to realize the read/write operations, i.e.
 fread/fwrite. This structure must have a path where to store files, as well as
 the maximum size the software can afford storing on the disk.
 
 Don't forget to check if the result is correct!
 
+This can also be achieved by just setting environment variables:
+
+\verbatim
+exoprt STARPU_DISK_SWAP=/tmp
+exoprt STARPU_DISK_SWAP_BACKEND=unistd
+exoprt STARPU_DISK_SWAP_SIZE=$((200*1024*1024))
+\endverbatim
+
 When the register function is called, StarPU will benchmark the disk. This can
 take some time.
 
@@ -54,8 +62,8 @@ The disk is unregistered during the starpu_shutdown().
 \section DiskFunctions Disk functions
 
 There are various ways to operate a disk memory node, described by the structure
-starpu_disk_ops. For instance, the variable #starpu_disk_stdio_ops
-uses fread/fwrite functions.
+starpu_disk_ops. For instance, the variable #starpu_disk_unistd_ops
+uses read/write functions.
 
 All structures are in \ref API_Out_Of_Core .
 

doc/doxygen/chapters/40environment_variables.doxy

@@ -565,9 +565,27 @@ that have a limited amount of memory.
 \anchor STARPU_LIMIT_CPU_MEM
 \addindex __env__STARPU_LIMIT_CPU_MEM
 This variable specifies the maximum number of megabytes that should be
-available to the application on each CPU device. This variable is
-intended to be used for experimental purposes as it emulates devices
-that have a limited amount of memory.
+available to the application on each CPU device.
+</dd>
+
+<dt>STARPU_DISK_SWAP</dt>
+<dd>
+This specifies a path where StarPU can push data when the main memory is getting
+full.
+</dd>
+
+<dt>STARPU_DISK_SWAP_BACKEND</dt>
+<dd>
+This specifies then backend to be used by StarPU to push data when the main
+memory is getting full. The default is unistd (i.e. using read/write functions),
+other values are stdio (i.e. using fread/fwrite), unistd_o_direct (i.e. using
+read/write with O_DIRECT), and leveldb (i.e. using a leveldb database).
+</dd>
+
+<dt>STARPU_DISK_SWAP_SIZE</dt>
+<dd>
+This specifies then size to be used by StarPU to push data when the main
+memory is getting full. The default is unlimited.
 </dd>
 
 <dt>STARPU_TRACE_BUFFER_SIZE</dt>

doc/doxygen/chapters/api/data_out_of_core.doxy

@@ -89,6 +89,7 @@ plug member of \p func will be passed \p parameter, and return a \c base which w
 SUCCESS: return the disk node. <br />
 FAIL: return an error code. <br />
 The \p size must be at least 1 MB !
+\p size being negative means infinite size.
 
 \fn void *starpu_disk_open(unsigned node, void *pos, size_t size)
 \ingroup API_Out_Of_Core

doc/doxygen/chapters/code/disk_compute.c

@@ -25,7 +25,7 @@
 #include <unistd.h>
 #include <math.h>
 
-#define NX (100)
+#define NX (1024)
 
 int main(int argc, char **argv)
 {
@@ -39,27 +39,21 @@ int main(int argc, char **argv)
 	int pid = getpid();
 	snprintf(pid_str, 16, "%d", pid);
 
-	char * base = "/tmp/";
+	const char *name_file_start = "STARPU_DISK_COMPUTE_DATA_";
+	const char *name_file_end = "STARPU_DISK_COMPUTE_DATA_RESULT_";
 
-	char * name_file_start = malloc(128*sizeof(char));
-	strcpy(name_file_start, "STARPU_DISK_COMPUTE_DATA_");
-	strcat(name_file_start, pid_str);
-
-	char * name_file_end = malloc(128*sizeof(char));
-	strcpy(name_file_end, "STARPU_DISK_COMPUTE_DATA_RESULT_");
-	strcat(name_file_end, pid_str);
-
-	char * path_file_start = malloc(128*sizeof(char));
+	char * path_file_start = malloc(strlen(base) + 1 + strlen(name_file_start) + 1);
 	strcpy(path_file_start, base);
+	strcat(path_file_start, "/");
 	strcat(path_file_start, name_file_start);
 
-	char * path_file_end = malloc(128*sizeof(char));
+	char * path_file_end = malloc(strlen(base) + 1 + strlen(name_file_end) + 1);
 	strcpy(path_file_end, base);
+	strcat(path_file_end, "/");
 	strcat(path_file_end, name_file_end);
 
-
 	/* register a disk */
-	int new_dd = starpu_disk_register(&starpu_disk_stdio_ops, (void *) base, 1024*1024*1);
+	int new_dd = starpu_disk_register(&starpu_disk_unistd_ops, (void *) base, 1024*1024*1);
 	/* can't write on /tmp/ */
 	if (new_dd == -ENOENT) goto enoent;
 	
@@ -88,7 +82,7 @@ int main(int argc, char **argv)
 	/* you create a file to store the vector ON the disk */
 	FILE * f = fopen(path_file_start, "wb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 
 	/* store it in the file */
 	fwrite(A, sizeof(int), NX, f);
@@ -100,7 +94,7 @@ int main(int argc, char **argv)
 	/* create a file to store result */
 	f = fopen(path_file_end, "wb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 
 	/* replace all datas by 0 */
 	fwrite(C, sizeof(int), NX, f);
@@ -113,7 +107,6 @@ int main(int argc, char **argv)
 	void * data = starpu_disk_open(dd, (void *) name_file_start, NX*sizeof(int));
 	void * data_result = starpu_disk_open(dd, (void *) name_file_end, NX*sizeof(int));
 
-
 	starpu_data_handle_t vector_handleA, vector_handleC;
 
 	/* register vector in starpu */
@@ -150,14 +143,12 @@ int main(int argc, char **argv)
 			try = 0;
 		}
 
-	starpu_free_flags(A, NX*sizeof(double), STARPU_MALLOC_COUNT);
-	starpu_free_flags(C, NX*sizeof(double), STARPU_MALLOC_COUNT);
+	starpu_free_flags(A, NX*sizeof(int), STARPU_MALLOC_COUNT);
+	starpu_free_flags(C, NX*sizeof(int), STARPU_MALLOC_COUNT);
 
 	unlink(path_file_start);
 	unlink(path_file_end);
 
-	free(name_file_start);
-	free(name_file_end);
 	free(path_file_start);
 	free(path_file_end);
 
@@ -172,7 +163,17 @@ int main(int argc, char **argv)
 
 enodev:
 	return 77;
+enoent2:
+	starpu_free_flags(A, NX*sizeof(int), STARPU_MALLOC_COUNT);
+	starpu_free_flags(C, NX*sizeof(int), STARPU_MALLOC_COUNT);
 enoent:
+	unlink(path_file_start);
+	unlink(path_file_end);
+
+	free(path_file_start);
+	free(path_file_end);
+
+	starpu_shutdown();
 	return 77;
 }
 //! [To be included. You should update doxygen if you see this text.]

doc/doxygen/chapters/code/disk_copy.c

@@ -43,11 +43,9 @@ int main(int argc, char **argv)
 	if (ret == -ENODEV) goto enodev;
 
 	/* register a disk */
-	int new_dd = starpu_disk_register(&starpu_disk_stdio_ops, (void *) "/tmp/", 1024*1024*200);
+	int new_dd = starpu_disk_register(&starpu_disk_unistd_ops, (void *) "/tmp/", 1024*1024*200);
 	/* can't write on /tmp/ */
 	if (new_dd == -ENOENT) goto enoent;
-	
-	unsigned dd = (unsigned) new_dd;
 
 	/* allocate two memory spaces */
 	starpu_malloc_flags((void **)&A, NX*sizeof(double), STARPU_MALLOC_COUNT);

include/starpu_disk.h

@@ -23,7 +23,7 @@
 /* list of functions to use on disk */
 struct starpu_disk_ops
 {
-	 void *  (*plug)   (void *parameter, size_t size);
+	 void *  (*plug)   (void *parameter, ssize_t size);
 	 void    (*unplug) (void *base);
 
 	 int    (*bandwidth)    (unsigned node);
@@ -64,6 +64,6 @@ void starpu_disk_close(unsigned node, void *obj, size_t size);
 
 void *starpu_disk_open(unsigned node, void *pos, size_t size);
 
-int starpu_disk_register(struct starpu_disk_ops *func, void *parameter, size_t size);
+int starpu_disk_register(struct starpu_disk_ops *func, void *parameter, ssize_t size);
 
 #endif /* __STARPU_DISK_H__ */

src/core/disk.c

@@ -53,10 +53,10 @@ static int size_register_list = 2;
 
 
 int
-starpu_disk_register(struct starpu_disk_ops * func, void *parameter, size_t size)
+starpu_disk_register(struct starpu_disk_ops * func, void *parameter, ssize_t size)
 {
 
-	STARPU_ASSERT_MSG(size >= SIZE_DISK_MIN,"Minimum disk size is %u Bytes ! (Here %u) \n", (int) SIZE_DISK_MIN, (int) size);
+	STARPU_ASSERT_MSG(size < 0 || size >= SIZE_DISK_MIN,"Minimum disk size is %u Bytes ! (Here %u) \n", (int) SIZE_DISK_MIN, (int) size);
 	/* register disk */
 	unsigned memory_node = _starpu_memory_node_register(STARPU_DISK_RAM, 0);
 
@@ -73,7 +73,8 @@ starpu_disk_register(struct starpu_disk_ops * func, void *parameter, size_t size
 	/* have a problem with the disk */
 	if(ret == 0)
 		return -ENOENT;
-	_starpu_memory_manager_set_global_memory_size(memory_node, size);
+	if (size >= 0)
+		_starpu_memory_manager_set_global_memory_size(memory_node, size);
 	return memory_node;
 }
 
@@ -359,3 +360,50 @@ _starpu_get_disk_flag(unsigned node)
 	int pos = get_location_with_node(node);
 	return disk_register_list[pos]->flag;
 }
+
+void
+_starpu_swap_init(void)
+{
+	char *backend;
+	char *path;
+	ssize_t size;
+	struct starpu_disk_ops *ops;
+	int dd;
+
+	path = getenv("STARPU_DISK_SWAP");
+	if (!path)
+		return;
+
+	backend = getenv("STARPU_DISK_SWAP_BACKEND");
+	if (!backend)
+		ops = &starpu_disk_unistd_ops;
+	else if (!strcmp(backend, "stdio"))
+		ops = &starpu_disk_stdio_ops;
+	else if (!strcmp(backend, "unistd"))
+		ops = &starpu_disk_unistd_ops;
+	else if (!strcmp(backend, "unistd_o_direct"))
+		ops = &starpu_disk_unistd_o_direct_ops;
+	else if (!strcmp(backend, "leveldb"))
+	{
+#ifdef STARPU_HAVE_LEVELDB
+		ops = &starpu_disk_leveldb_ops;
+#else
+		_STARPU_DISP("Warnin: leveldb support is not compiled in, could not enable disk swap");
+		return;
+#endif
+	}
+	else
+	{
+		_STARPU_DISP("Warning: unknown disk swap backend %s, could not enable disk swap", backend);
+		return;
+	}
+
+	size = starpu_get_env_number_default("STARPU_DISK_SWAP_SIZE", -1);
+
+	dd = starpu_disk_register(ops, path, size);
+	if (dd < 0)
+	{
+		_STARPU_DISP("Warning: could not enable disk swap %s on %s with size %ld, could not enable disk swap", backend, path, (long) size);
+		return;
+	}
+}

src/core/disk.h

@@ -62,6 +62,8 @@ int _starpu_get_disk_flag(unsigned node);
 
 void _starpu_disk_unregister(void);
 
+void _starpu_swap_init(void);
+
 #ifdef __cplusplus
 }
 #endif

src/core/disk_ops/disk_leveldb.cpp

@@ -234,7 +234,7 @@ starpu_leveldb_full_write (void * base, void * obj, void * ptr, size_t size)
 
 /* create a new copy of parameter == base */
 static void * 
-starpu_leveldb_plug (void *parameter, size_t size STARPU_ATTRIBUTE_UNUSED)
+starpu_leveldb_plug (void *parameter, ssize_t size STARPU_ATTRIBUTE_UNUSED)
 {
 	struct starpu_leveldb_base * tmp = (struct starpu_leveldb_base *) malloc(sizeof(struct starpu_leveldb_base));
 	STARPU_ASSERT(tmp != NULL);

src/core/disk_ops/disk_stdio.c

@@ -253,7 +253,7 @@ static int starpu_stdio_full_write(void * base STARPU_ATTRIBUTE_UNUSED, void * o
 }
 
 /* create a new copy of parameter == base */
-static void *starpu_stdio_plug(void *parameter, size_t size STARPU_ATTRIBUTE_UNUSED)
+static void *starpu_stdio_plug(void *parameter, ssize_t size STARPU_ATTRIBUTE_UNUSED)
 {
 	char * tmp = malloc(sizeof(char)*(strlen(parameter)+1));
 	STARPU_ASSERT(tmp != NULL);

src/core/disk_ops/disk_unistd_o_direct.c

@@ -80,7 +80,7 @@ starpu_unistd_o_direct_write (void *base STARPU_ATTRIBUTE_UNUSED, void *obj, con
 
 /* create a new copy of parameter == base */
 static void * 
-starpu_unistd_o_direct_plug (void *parameter, size_t size STARPU_ATTRIBUTE_UNUSED)
+starpu_unistd_o_direct_plug (void *parameter, ssize_t size)
 {
 	starpu_malloc_set_align(getpagesize());
 

src/core/disk_ops/unistd/disk_unistd_global.c

@@ -311,7 +311,7 @@ starpu_unistd_global_full_write (void * base STARPU_ATTRIBUTE_UNUSED, void * obj
 
 /* create a new copy of parameter == base */
  void * 
-starpu_unistd_global_plug (void *parameter, size_t size STARPU_ATTRIBUTE_UNUSED)
+starpu_unistd_global_plug (void *parameter, ssize_t size STARPU_ATTRIBUTE_UNUSED)
 {
 	char * tmp = malloc(sizeof(char)*(strlen(parameter)+1));
 	STARPU_ASSERT(tmp != NULL);

src/core/disk_ops/unistd/disk_unistd_global.h

@@ -37,7 +37,7 @@ void * starpu_unistd_global_open (struct starpu_unistd_global_obj * obj, void *b
 void starpu_unistd_global_close (void *base, void *obj, size_t size);
 int starpu_unistd_global_read (void *base, void *obj, void *buf, off_t offset, size_t size);
 int starpu_unistd_global_write (void *base, void *obj, const void *buf, off_t offset, size_t size);
-void * starpu_unistd_global_plug (void *parameter, size_t size);
+void * starpu_unistd_global_plug (void *parameter, ssize_t size);
 void starpu_unistd_global_unplug (void *base);
 int get_unistd_global_bandwidth_between_disk_and_main_ram(unsigned node);
 void* starpu_unistd_global_async_read (void *base, void *obj, void *buf, off_t offset, size_t size);

src/core/workers.c

@@ -1186,6 +1186,9 @@ int starpu_initialize(struct starpu_conf *user_conf, int *argc, char ***argv)
 		return ret;
 	}
 
+	/* Allocate swap, if any */
+	_starpu_swap_init();
+
 	/* We need to store the current task handled by the different
 	 * threads */
 	_starpu_initialize_current_task_key();

tests/disk/disk_compute.c

@@ -108,7 +108,7 @@ int dotest(struct starpu_disk_ops *ops, char *base)
 	/* create a file to store result */
 	f = fopen(path_file_end, "wb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 
 	/* replace all datas by 0 */
 	fwrite(C, sizeof(int), NX, f);
@@ -150,7 +150,7 @@ int dotest(struct starpu_disk_ops *ops, char *base)
 	/* check results */
 	f = fopen(path_file_end, "rb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 	/* take datas */
 	int size = fread(C, sizeof(int), NX, f);
 

tests/disk/disk_copy.c

@@ -69,8 +69,6 @@ int dotest(struct starpu_disk_ops *ops, void *param)
 	/* can't write on /tmp/ */
 	if (new_dd == -ENOENT) goto enoent;
 
-	unsigned dd = (unsigned) new_dd;
-
 	/* allocate two memory spaces */
 	starpu_malloc_flags((void **)&A, NX*sizeof(double), STARPU_MALLOC_COUNT);
 	starpu_malloc_flags((void **)&F, NX*sizeof(double), STARPU_MALLOC_COUNT);

tests/disk/disk_pack.c

@@ -134,7 +134,7 @@ int dotest(struct starpu_disk_ops *ops, char *base)
 	/* create a file to store result */
 	f = fopen(path_file_end, "wb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 
 	/* replace all datas by 0 */
 	fwrite(C, sizeof(int), NX, f);
@@ -180,7 +180,7 @@ int dotest(struct starpu_disk_ops *ops, char *base)
 	/* check results */	
 	f = fopen(path_file_end, "rb+");
 	if (f == NULL)
-		goto enoent;
+		goto enoent2;
 	/* take datas */
 	int size = fread(C, sizeof(int), NX, f);