|
|
@@ -1,124 +1,44 @@
|
|
|
@c -*-texinfo-*-
|
|
|
|
|
|
@c This file is part of the StarPU Handbook.
|
|
|
-@c Copyright (C) 2011, 2012 Institut National de Recherche en Informatique et Automatique
|
|
|
+@c Copyright (C) 2011--2013 Institut National de Recherche en Informatique et Automatique
|
|
|
@c See the file starpu.texi for copying conditions.
|
|
|
|
|
|
@cindex Scheduling Context Hypervisor
|
|
|
|
|
|
+@menu
|
|
|
+* What is the Hypervisor::
|
|
|
+* Start the Hypervisor::
|
|
|
+* Interrogate the runtime::
|
|
|
+* Trigger the Hypervisor::
|
|
|
+* Resizing strategies::
|
|
|
+@end menu
|
|
|
+
|
|
|
+@node What is the Hypervisor
|
|
|
+@section What is the Hypervisor
|
|
|
StarPU proposes a platform for constructing Scheduling Contexts, for deleting and modifying them dynamically.
|
|
|
A parallel kernel, can thus be isolated into a scheduling context and interferences between several parallel kernels are avoided.
|
|
|
If the user knows exactly how many workers each scheduling context needs, he can assign them to the contexts at their creation time or modify them during the execution of the program.
|
|
|
|
|
|
-The Scheduling Context Hypervisor Plugin is available for the users who do not dispose of a regular parallelism, who cannot know in advance the exact size of the context and need to resize the contexts according to the behavior of the parallel kernel.
|
|
|
+The Scheduling Context Hypervisor Plugin is available for the users who do not dispose of a regular parallelism, who cannot know in advance the exact size of the context and need to resize the contexts according to the behavior of the parallel kernels.
|
|
|
The Hypervisor receives information from StarPU concerning the execution of the tasks, the efficiency of the resources, etc. and it decides accordingly when and how the contexts can be resized.
|
|
|
Basic strategies of resizing scheduling contexts already exist but a platform for implementing additional custom ones is available.
|
|
|
|
|
|
-@menu
|
|
|
-* Managing the hypervisor:: Initialize the hypervisor
|
|
|
-* Registering Scheduling Contexts to the hypervisor:: Contexts have to register to the hypervisor
|
|
|
-* The user's input in the resizing process:: The user can help the hypervisor decide how to resize
|
|
|
-* Resizing strategies:: Several resizing strategies are proposed
|
|
|
-* Performance Counters:: StarPU provides information to the Hypervisor through performance counters
|
|
|
-* Defining a new hypervisor policy:: New Policies can be implemented
|
|
|
-@end menu
|
|
|
-
|
|
|
-@node Managing the hypervisor
|
|
|
-@section Managing the hypervisor
|
|
|
-There is a single hypervisor that is in charge of resizing contexts and the resizing strategy is chosen at the initialization of the hypervisor. A single resize can be done at a time.
|
|
|
-
|
|
|
-@deftypefun {struct starpu_sched_ctx_performance_counters *} sched_ctx_hypervisor_init ({struct sched_ctx_hypervisor_policy *} @var{policy})
|
|
|
-Initializes the hypervisor to use the strategy provided as parameter and creates the performance counters (see @pxref{Performance Counters}).
|
|
|
-These performance counters represent actually some callbacks that will be used by the contexts to notify the information needed by the hypervisor.
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-Note: The Hypervisor is actually a worker that takes this role once certain conditions trigger the resizing process (there is no additional thread assigned to the hypervisor).
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_shutdown (void)
|
|
|
-The hypervisor and all information is freed. There is no synchronization between this function and starpu_shutdown. Thus, this should be done after starpu_shutdown(),
|
|
|
-because the performance counters will still need allocated callback functions.
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@node Registering Scheduling Contexts to the hypervisor
|
|
|
-@section Registering Scheduling Contexts to the hypervisor
|
|
|
-Scheduling Contexts that have to be resized by the hypervisor must be first registered to the hypervisor. Whenever we want to exclude contexts from the resizing process we have to unregister them from the hypervisor.
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_register_ctx (unsigned @var{sched_ctx}, double @var{total_flops})
|
|
|
-Register the context to the hypervisor, and indicate the number of flops the context will execute (needed for Gflops rate based strategy @pxref{Resizing strategies} or any other custom strategy needing it, for the others we can pass 0.0)
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_unregister_ctx (unsigned @var{sched_ctx})
|
|
|
-Unregister the context from the hypervisor
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@node The user's input in the resizing process
|
|
|
-@section The user's input in the resizing process
|
|
|
-The user can totally forbid the resizing of a certain context or can then change his mind and allow it (in this case the resizing is managed by the hypervisor, that can forbid it or allow it)
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_stop_resize (unsigned @var{sched_ctx})
|
|
|
-Forbid resizing of a context
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_start_resize (unsigned @var{sched_ctx})
|
|
|
-Allow resizing of a context
|
|
|
-@end deftypefun
|
|
|
+@node Start the Hypervisor
|
|
|
+@section Start the Hypervisor
|
|
|
+The Hypervisor must be initialised once at the beging of the application. At this point a resizing policy should be indicated. This strategy depends on the information the application is able to provide to the hypervisor as well
|
|
|
+as on the accuracy needed for the resizing procedure. For exemple, the application may be able to provide an estimation of the workload of the contexts. In this situation the hypervisor may decide what resources the contexts need.
|
|
|
+However, if no information is provided the hypervisor evaluates the behavior of the resources and of the application and makes a guess about the future.
|
|
|
+The hypervisor resizes only the registered contexts.
|
|
|
|
|
|
-The user can then provide information to the hypervisor concerning the conditions of resizing.
|
|
|
+@node Interrogate the runtime
|
|
|
+@section Interrrogate the runtime
|
|
|
+The runtime provides the hypervisor with information concerning the behavior of the resources and the application. This is done by using the performance_counters, some callbacks indicating when the resources are idle or not efficient, when the application submits tasks or when it becames to slow.
|
|
|
|
|
|
-@deftypefun void sched_ctx_hypervisor_ioctl (unsigned @var{sched_ctx}, ...)
|
|
|
-Inputs conditions to the context @code{sched_ctx} with the following arguments. The argument list must be zero-terminated.
|
|
|
+@node Trigger the Hypervisor
|
|
|
+@section Trigger the Hypervisor
|
|
|
+The resizing is triggered either when the application requires it or when the initials distribution of resources alters the performance of the application( the application is to slow or the resource are idle for too long time, threashold indicated by the user). When this happens different resizing strategy are applied that target minimising the total execution of the application, the instant speed or the idle time of the resources.
|
|
|
|
|
|
-@defmac HYPERVISOR_MAX_IDLE
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 3 arguments:
|
|
|
-an array of int for the workerids to apply the condition, an int to indicate the size of the array, and a double value indicating
|
|
|
-the maximum idle time allowed for a worker before the resizing process should be triggered
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_PRIORITY
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 3 arguments:
|
|
|
-an array of int for the workerids to apply the condition, an int to indicate the size of the array, and an int value indicating
|
|
|
-the priority of the workers previously mentioned.
|
|
|
-The workers with the smallest priority are moved the first.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_MIN_WORKERS
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument(int) indicating
|
|
|
-the minimum number of workers a context should have, underneath this limit the context cannot execute.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_MAX_WORKERS
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument(int) indicating
|
|
|
-the maximum number of workers a context should have, above this limit the context would not be able to scale
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_GRANULARITY
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument(int) indicating
|
|
|
-the granularity of the resizing process (the number of workers should be moved from the context once it is resized)
|
|
|
-This parameter is ignore for the Gflops rate based strategy @pxref{Resizing strategies}, the number of workers that have to be moved is calculated by the strategy.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_FIXED_WORKERS
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 2 arguments:
|
|
|
-an array of int for the workerids to apply the condition and an int to indicate the size of the array.
|
|
|
-These workers are not allowed to be moved from the context.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_MIN_TASKS
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument (int)
|
|
|
-that indicated the minimum number of tasks that have to be executed before the context could be resized.
|
|
|
-This parameter is ignored for the Application Driven strategy @pxref{Resizing strategies} where the user indicates exactly when the resize should be done.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_NEW_WORKERS_MAX_IDLE
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument, a double value indicating
|
|
|
-the maximum idle time allowed for workers that have just been moved from other contexts in the current context.
|
|
|
-@end defmac
|
|
|
-
|
|
|
-@defmac HYPERVISOR_TIME_TO_APPLY
|
|
|
-This macro is used when calling sched_ctx_hypervisor_ioctl and must be followed by 1 argument (int) indicating the tag
|
|
|
-an executed task should have such that this configuration should be taken into account.
|
|
|
-@end defmac
|
|
|
-@end deftypefun
|
|
|
@node Resizing strategies
|
|
|
@section Resizing strategies
|
|
|
|
|
|
@@ -205,190 +125,4 @@ starpu_insert_task(&codelet,
|
|
|
@end smallexample
|
|
|
@end cartouche
|
|
|
|
|
|
-@node Performance Counters
|
|
|
-@section Performance Counters
|
|
|
-
|
|
|
-The Scheduling Context Hypervisor Plugin provides a series of performance counters to StarPU. By incrementing them, StarPU can help the hypervisor in the resizing decision making process.
|
|
|
-
|
|
|
-@deftp {Data Type} {struct starpu_sched_ctx_performance_counters}
|
|
|
-@anchor{struct starpu_sched_ctx_performance_counters}
|
|
|
-
|
|
|
-@table @asis
|
|
|
-@item @code{void (*notify_idle_cycle)(unsigned sched_ctx_id, int worker, double idle_time)}
|
|
|
-Informs the hypervisor for how long a worker has been idle in the specified context
|
|
|
-@item @code{void (*notify_idle_end)(unsigned sched_ctx_id, int worker)}
|
|
|
-Informs the hypervisor that after a period of idle, the worker has just executed a task in the specified context.
|
|
|
-The idle counter it though reset.
|
|
|
-@item @code{void (*notify_pushed_task)(unsigned sched_ctx_id, int worker)}
|
|
|
-Notifies the hypervisor a task has been scheduled on the queue of the worker corresponding to the specified context
|
|
|
-@item @code{void (*notify_poped_task)(unsigned sched_ctx_id, int worker, double flops)}
|
|
|
-Informs the hypervisor a task executing a specified number of instructions has been poped from the worker
|
|
|
-@item @code{void (*notify_post_exec_hook)(unsigned sched_ctx_id, int taskid)}
|
|
|
-Notifies the hypervisor a task has just been executed
|
|
|
-
|
|
|
-@end table
|
|
|
-@end deftp
|
|
|
-
|
|
|
-TODO maybe they should be hidden to the user
|
|
|
-
|
|
|
-@node Defining a new hypervisor policy
|
|
|
-@section Defining a new hypervisor policy
|
|
|
-
|
|
|
-@menu
|
|
|
-* Hypervisor Policy API:: Hypervisor Policy API
|
|
|
-* Hypervisor example::
|
|
|
-@end menu
|
|
|
-
|
|
|
-@node Hypervisor Policy API
|
|
|
-@subsection Hypervisor Policy API
|
|
|
-
|
|
|
-While Scheduling Context Hypervisor Plugin comes with a variety of resizing policies (@pxref{Resizing strategies}),
|
|
|
-it may sometimes be desirable to implement custom
|
|
|
-policies to address specific problems. The API described below allows
|
|
|
-users to write their own resizing policy.
|
|
|
-
|
|
|
-@deftp {Data Type} {struct sched_ctx_hypervisor_policy}
|
|
|
-This structure contains all the methods that implement a hypervisor resizing policy.
|
|
|
-
|
|
|
-@table @asis
|
|
|
-@item @code{const char* name}
|
|
|
-Indicates the name of the policy, if there is not a custom policy, the policy corresponding to this name will be used by the hypervisor
|
|
|
-@item @code{unsigned custom}
|
|
|
-Indicates whether the policy is custom or not
|
|
|
-@item @code{void (*handle_idle_cycle)(unsigned sched_ctx_id, int worker)}
|
|
|
-It is called whenever the indicated worker executes another idle cycle in @code{sched_ctx}
|
|
|
-@item @code{void (*handle_pushed_task)(unsigned sched_ctx_id, int worker)}
|
|
|
-It is called whenever a task is pushed on the worker's queue corresponding to the context @code{sched_ctx}
|
|
|
-@item @code{void (*handle_poped_task)(unsigned sched_ctx_id, int worker)}
|
|
|
-It is called whenever a task is poped from the worker's queue corresponding to the context @code{sched_ctx}
|
|
|
-@item @code{void (*handle_idle_end)(unsigned sched_ctx_id, int worker)}
|
|
|
-It is called whenever a task is executed on the indicated worker and context after a long period of idle time
|
|
|
-@item @code{void (*handle_post_exec_hook)(unsigned sched_ctx_id, struct starpu_htbl32_node* resize_requests, int task_tag)}
|
|
|
-It is called whenever a tag task has just been executed. The table of resize requests is provided as well as the tag
|
|
|
-@end table
|
|
|
-@end deftp
|
|
|
-
|
|
|
-The Hypervisor provides also a structure with configuration information of each context, which can be used to construct new resize strategies.
|
|
|
-
|
|
|
-@deftp {Data Type} {struct sched_ctx_hypervisor_policy_config }
|
|
|
-This structure contains all configuration information of a context
|
|
|
-
|
|
|
-@table @asis
|
|
|
-@item @code{int min_nworkers}
|
|
|
-Indicates the minimum number of workers needed by the context
|
|
|
-@item @code{int max_nworkers}
|
|
|
-Indicates the maximum number of workers needed by the context
|
|
|
-@item @code{int granularity}
|
|
|
-Indicates the workers granularity of the context
|
|
|
-@item @code{int priority[STARPU_NMAXWORKERS]}
|
|
|
-Indicates the priority of each worker in the context
|
|
|
-@item @code{double max_idle[STARPU_NMAXWORKERS]}
|
|
|
-Indicates the maximum idle time accepted before a resize is triggered
|
|
|
-@item @code{int fixed_workers[STARPU_NMAXWORKERS]}
|
|
|
-Indicates which workers can be moved and which ones are fixed
|
|
|
-@item @code{double new_workers_max_idle}
|
|
|
-Indicates the maximum idle time accepted before a resize is triggered for the workers that just arrived in the new context
|
|
|
-@end table
|
|
|
-@end deftp
|
|
|
-
|
|
|
-Additionally, the hypervisor provides a structure with information obtained from StarPU by means of the performance counters
|
|
|
-
|
|
|
-
|
|
|
-@deftp {Data Type} {struct sched_ctx_hypervisor_wrapper}
|
|
|
-This structure is a wrapper of the contexts available in StarPU
|
|
|
-and contains all information about a context obtained by incrementing the performance counters
|
|
|
-
|
|
|
-@table @asis
|
|
|
-@item @code{unsigned sched_ctx}
|
|
|
-The context wrapped
|
|
|
-@item @code{struct sched_ctx_hypervisor_policy_config *config}
|
|
|
-The corresponding resize configuration
|
|
|
-@item @code{double current_idle_time[STARPU_NMAXWORKERS]}
|
|
|
-The idle time counter of each worker of the context
|
|
|
-@item @code{int pushed_tasks[STARPU_NMAXWORKERS]}
|
|
|
-The number of pushed tasks of each worker of the context
|
|
|
-@item @code{int poped_tasks[STARPU_NMAXWORKERS]}
|
|
|
-The number of poped tasks of each worker of the context
|
|
|
-@item @code{double total_flops}
|
|
|
-The total number of flops to execute by the context
|
|
|
-@item @code{double total_elapsed_flops[STARPU_NMAXWORKERS]}
|
|
|
-The number of flops executed by each workers of the context
|
|
|
-@item @code{double elapsed_flops[STARPU_NMAXWORKERS]}
|
|
|
-The number of flops executed by each worker of the context from last resize
|
|
|
-@item @code{double remaining_flops}
|
|
|
-The number of flops that still have to be executed by the workers in the context
|
|
|
-@item @code{double start_time}
|
|
|
-The time when he started executed
|
|
|
-@item @code{struct sched_ctx_hypervisor_resize_ack resize_ack}
|
|
|
-The structure confirming the last resize finished and a new one can be done
|
|
|
-@end table
|
|
|
-@end deftp
|
|
|
-
|
|
|
-@deftp {Data Type} {struct sched_ctx_hypervisor_resize_ack}
|
|
|
-This structures checks if the workers moved to another context are actually taken into account in that context
|
|
|
-@table @asis
|
|
|
-@item @code{int receiver_sched_ctx}
|
|
|
-The context receiving the new workers
|
|
|
-@item @code{int *moved_workers}
|
|
|
-The workers moved to the receiver context
|
|
|
-@item @code{int nmoved_workers}
|
|
|
-The number of workers moved
|
|
|
-@item @code{int *acked_workers}
|
|
|
-If the value corresponding to a worker is 1, this one is taken into account in the new context if 0 not yet
|
|
|
-@end table
|
|
|
-@end deftp
|
|
|
-
|
|
|
-The following functions can be used in the resizing strategies.
|
|
|
-
|
|
|
-@deftypefun void sched_ctx_hypervisor_move_workers (unsigned @var{sender_sched_ctx}, unsigned @var{receiver_sched_ctx}, {int *}@var{workers_to_move}, unsigned @var{nworkers_to_move}, unsigned @var{now});
|
|
|
-Moves workers from one context to another
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun {struct sched_ctx_hypervisor_policy_config *} sched_ctx_hypervisor_get_config (unsigned @var{sched_ctx});
|
|
|
-Returns the configuration structure of a context
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun {int *} sched_ctx_hypervisor_get_sched_ctxs ();
|
|
|
-Gets the contexts managed by the hypervisor
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun int sched_ctx_hypervisor_get_nsched_ctxs ();
|
|
|
-Gets the number of contexts managed by the hypervisor
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun {struct sched_ctx_hypervisor_wrapper *} sched_ctx_hypervisor_get_wrapper (unsigned @var{sched_ctx});
|
|
|
-Returns the wrapper corresponding the context @code{sched_ctx}
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun double sched_ctx_hypervisor_get_elapsed_flops_per_sched_ctx ({struct sched_ctx_hypervisor_wrapper *} @var{sc_w});
|
|
|
-Returns the flops of a context elapsed from the last resize
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@deftypefun {char *} sched_ctx_hypervisor_get_policy ();
|
|
|
-Returns the name of the resizing policy the hypervisor uses
|
|
|
-@end deftypefun
|
|
|
-
|
|
|
-@node Hypervisor example
|
|
|
-@subsection Hypervisor example
|
|
|
-
|
|
|
-@cartouche
|
|
|
-@smallexample
|
|
|
-
|
|
|
-struct sched_ctx_hypervisor_policy dummy_policy =
|
|
|
-@{
|
|
|
- .handle_poped_task = dummy_handle_poped_task,
|
|
|
- .handle_pushed_task = dummy_handle_pushed_task,
|
|
|
- .handle_idle_cycle = dummy_handle_idle_cycle,
|
|
|
- .handle_idle_end = dummy_handle_idle_end,
|
|
|
- .handle_post_exec_hook = dummy_handle_post_exec_hook,
|
|
|
- .custom = 1,
|
|
|
- .name = "dummy"
|
|
|
-@};
|
|
|
-
|
|
|
-@end smallexample
|
|
|
-@end cartouche
|
|
|
|
|
|
-@c Local Variables:
|
|
|
-@c TeX-master: "../starpu.texi"
|
|
|
-@c ispell-local-dictionary: "american"
|
|
|
-@c End:
|
Andra Hugo