create a hierarchy of directories to store allocated files, to avoid issues with systems where the number of files in the same directory is limited. A depth of 2 gets a billion of files with a thousand per directory, that should be enough

src/common/utils.c

@@ -172,6 +172,41 @@ char *_starpu_mktemp(const char *directory, int flags, int *fd)
 	return baseCpy;
 }
 
+char *_starpu_mktemp_many(const char *directory, int depth, int flags, int *fd)
+{
+	size_t len = strlen(directory);
+	char path[len + depth*4 + 1];
+	int i;
+
+	memcpy(path, directory, len);
+	for (i = 0; i < depth; i++)
+	{
+		int r = starpu_lrand48();
+		path[len + i*4 + 0] = '/';
+		path[len + i*4 + 1] = '0' + (r/1)%10;
+		path[len + i*4 + 2] = '0' + (r/10)%10;
+		path[len + i*4 + 3] = '0' + (r/100)%10;
+		path[len + i*4 + 4] = 0;
+		if (mkdir(path, 0777) < 0 && errno != EEXIST)
+		{
+			_STARPU_DISP("Could not create temporary directory '%s', mkdir failed with error '%s'\n", path, strerror(errno));
+			return NULL;
+		}
+	}
+	return _starpu_mktemp(path, flags, fd);
+}
+
+void _starpu_rmtemp_many(char *path, int depth)
+{
+	int i;
+	for (i = 0; i < depth; i++)
+	{
+		path = dirname(path);
+		if (rmdir(path) < 0 && errno != ENOTEMPTY && errno != EBUSY)
+			_STARPU_DISP("Could not remove temporary directory '%s', rmdir failed with error '%s'\n", path, strerror(errno));
+	}
+}
+
 int _starpu_ftruncate(FILE *file)
 {
 	return ftruncate(fileno(file), 0);

src/common/utils.h

@@ -125,6 +125,10 @@
 int _starpu_mkpath(const char *s, mode_t mode);
 void _starpu_mkpath_and_check(const char *s, mode_t mode);
 char *_starpu_mktemp(const char *directory, int flags, int *fd);
+/* This version creates a hierarchy of n temporary directories, useful when
+ * creating a lot of temporary files to be stored in the same place */
+char *_starpu_mktemp_many(const char *directory, int depth, int flags, int *fd);
+void _starpu_rmtemp_many(char *path, int depth);
 int _starpu_ftruncate(FILE *file);
 int _starpu_frdlock(FILE *file);
 int _starpu_frdunlock(FILE *file);

src/core/disk_ops/disk_stdio.c

@@ -40,6 +40,8 @@
 #define O_BINARY 0
 #endif
 
+#define TEMP_HIERARCHY_DEPTH 2
+
 /* ------------------- use STDIO to write on disk -------------------  */
 
 struct starpu_stdio_obj
@@ -58,7 +60,7 @@ static void *starpu_stdio_alloc(void *base, size_t size)
 	STARPU_ASSERT(obj != NULL);
 
 	int id;
-	char *baseCpy = _starpu_mktemp(base, O_RDWR | O_BINARY, &id);
+	char *baseCpy = _starpu_mktemp_many(base, TEMP_HIERARCHY_DEPTH, O_RDWR | O_BINARY, &id);
 
 	/* fail */
 	if (!baseCpy)
@@ -120,6 +122,7 @@ static void starpu_stdio_free(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, siz
 	fclose(tmp->file);
 	close(tmp->descriptor);
 	unlink(tmp->path);
+	_starpu_rmtemp_many(tmp->path, TEMP_HIERARCHY_DEPTH);
 
 	free(tmp->path);
 	free(tmp);

src/core/disk_ops/unistd/disk_unistd_global.c

@@ -48,6 +48,8 @@
 #  define MEM_SIZE 1
 #endif
 
+#define TEMP_HIERARCHY_DEPTH 2
+
 /* TODO: on Linux, use io_submit */
 
 /* ------------------- use UNISTD to write on disk -------------------  */
@@ -56,7 +58,7 @@
 void *starpu_unistd_global_alloc(struct starpu_unistd_global_obj *obj, void *base, size_t size)
 {
 	int id;
-	char *baseCpy = _starpu_mktemp(base, obj->flags, &id);
+	char *baseCpy = _starpu_mktemp_many(base, TEMP_HIERARCHY_DEPTH, obj->flags, &id);
 
 	/* fail */
 	if (!baseCpy)
@@ -103,6 +105,7 @@ void starpu_unistd_global_free(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, si
 
 	close(tmp->descriptor);
 	unlink(tmp->path);
+	_starpu_rmtemp_many(tmp->path, TEMP_HIERARCHY_DEPTH);
 
 	free(tmp->path);
 	free(tmp);