sched_rt.c 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. /*
  2. * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
  3. * policies)
  4. */
  5. /*
  6. * Update the current task's runtime statistics. Skip current tasks that
  7. * are not in our scheduling class.
  8. */
  9. static inline void update_curr_rt(struct rq *rq, u64 now)
  10. {
  11. struct task_struct *curr = rq->curr;
  12. u64 delta_exec;
  13. if (!task_has_rt_policy(curr))
  14. return;
  15. delta_exec = now - curr->se.exec_start;
  16. if (unlikely((s64)delta_exec < 0))
  17. delta_exec = 0;
  18. if (unlikely(delta_exec > curr->se.exec_max))
  19. curr->se.exec_max = delta_exec;
  20. curr->se.sum_exec_runtime += delta_exec;
  21. curr->se.exec_start = now;
  22. }
  23. static void
  24. enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
  25. {
  26. struct prio_array *array = &rq->rt.active;
  27. list_add_tail(&p->run_list, array->queue + p->prio);
  28. __set_bit(p->prio, array->bitmap);
  29. }
  30. /*
  31. * Adding/removing a task to/from a priority array:
  32. */
  33. static void
  34. dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep, u64 now)
  35. {
  36. struct prio_array *array = &rq->rt.active;
  37. update_curr_rt(rq, now);
  38. list_del(&p->run_list);
  39. if (list_empty(array->queue + p->prio))
  40. __clear_bit(p->prio, array->bitmap);
  41. }
  42. /*
  43. * Put task to the end of the run list without the overhead of dequeue
  44. * followed by enqueue.
  45. */
  46. static void requeue_task_rt(struct rq *rq, struct task_struct *p)
  47. {
  48. struct prio_array *array = &rq->rt.active;
  49. list_move_tail(&p->run_list, array->queue + p->prio);
  50. }
  51. static void
  52. yield_task_rt(struct rq *rq, struct task_struct *p)
  53. {
  54. requeue_task_rt(rq, p);
  55. }
  56. /*
  57. * Preempt the current task with a newly woken task if needed:
  58. */
  59. static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
  60. {
  61. if (p->prio < rq->curr->prio)
  62. resched_task(rq->curr);
  63. }
  64. static struct task_struct * pick_next_task_rt(struct rq *rq, u64 now)
  65. {
  66. struct prio_array *array = &rq->rt.active;
  67. struct task_struct *next;
  68. struct list_head *queue;
  69. int idx;
  70. idx = sched_find_first_bit(array->bitmap);
  71. if (idx >= MAX_RT_PRIO)
  72. return NULL;
  73. queue = array->queue + idx;
  74. next = list_entry(queue->next, struct task_struct, run_list);
  75. next->se.exec_start = now;
  76. return next;
  77. }
  78. static void put_prev_task_rt(struct rq *rq, struct task_struct *p, u64 now)
  79. {
  80. update_curr_rt(rq, now);
  81. p->se.exec_start = 0;
  82. }
  83. /*
  84. * Load-balancing iterator. Note: while the runqueue stays locked
  85. * during the whole iteration, the current task might be
  86. * dequeued so the iterator has to be dequeue-safe. Here we
  87. * achieve that by always pre-iterating before returning
  88. * the current task:
  89. */
  90. static struct task_struct *load_balance_start_rt(void *arg)
  91. {
  92. struct rq *rq = arg;
  93. struct prio_array *array = &rq->rt.active;
  94. struct list_head *head, *curr;
  95. struct task_struct *p;
  96. int idx;
  97. idx = sched_find_first_bit(array->bitmap);
  98. if (idx >= MAX_RT_PRIO)
  99. return NULL;
  100. head = array->queue + idx;
  101. curr = head->prev;
  102. p = list_entry(curr, struct task_struct, run_list);
  103. curr = curr->prev;
  104. rq->rt.rt_load_balance_idx = idx;
  105. rq->rt.rt_load_balance_head = head;
  106. rq->rt.rt_load_balance_curr = curr;
  107. return p;
  108. }
  109. static struct task_struct *load_balance_next_rt(void *arg)
  110. {
  111. struct rq *rq = arg;
  112. struct prio_array *array = &rq->rt.active;
  113. struct list_head *head, *curr;
  114. struct task_struct *p;
  115. int idx;
  116. idx = rq->rt.rt_load_balance_idx;
  117. head = rq->rt.rt_load_balance_head;
  118. curr = rq->rt.rt_load_balance_curr;
  119. /*
  120. * If we arrived back to the head again then
  121. * iterate to the next queue (if any):
  122. */
  123. if (unlikely(head == curr)) {
  124. int next_idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1);
  125. if (next_idx >= MAX_RT_PRIO)
  126. return NULL;
  127. idx = next_idx;
  128. head = array->queue + idx;
  129. curr = head->prev;
  130. rq->rt.rt_load_balance_idx = idx;
  131. rq->rt.rt_load_balance_head = head;
  132. }
  133. p = list_entry(curr, struct task_struct, run_list);
  134. curr = curr->prev;
  135. rq->rt.rt_load_balance_curr = curr;
  136. return p;
  137. }
  138. static int
  139. load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
  140. unsigned long max_nr_move, unsigned long max_load_move,
  141. struct sched_domain *sd, enum cpu_idle_type idle,
  142. int *all_pinned, unsigned long *load_moved)
  143. {
  144. int this_best_prio, best_prio, best_prio_seen = 0;
  145. int nr_moved;
  146. struct rq_iterator rt_rq_iterator;
  147. best_prio = sched_find_first_bit(busiest->rt.active.bitmap);
  148. this_best_prio = sched_find_first_bit(this_rq->rt.active.bitmap);
  149. /*
  150. * Enable handling of the case where there is more than one task
  151. * with the best priority. If the current running task is one
  152. * of those with prio==best_prio we know it won't be moved
  153. * and therefore it's safe to override the skip (based on load)
  154. * of any task we find with that prio.
  155. */
  156. if (busiest->curr->prio == best_prio)
  157. best_prio_seen = 1;
  158. rt_rq_iterator.start = load_balance_start_rt;
  159. rt_rq_iterator.next = load_balance_next_rt;
  160. /* pass 'busiest' rq argument into
  161. * load_balance_[start|next]_rt iterators
  162. */
  163. rt_rq_iterator.arg = busiest;
  164. nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move,
  165. max_load_move, sd, idle, all_pinned, load_moved,
  166. this_best_prio, best_prio, best_prio_seen,
  167. &rt_rq_iterator);
  168. return nr_moved;
  169. }
  170. static void task_tick_rt(struct rq *rq, struct task_struct *p)
  171. {
  172. /*
  173. * RR tasks need a special form of timeslice management.
  174. * FIFO tasks have no timeslices.
  175. */
  176. if (p->policy != SCHED_RR)
  177. return;
  178. if (--p->time_slice)
  179. return;
  180. p->time_slice = static_prio_timeslice(p->static_prio);
  181. set_tsk_need_resched(p);
  182. /* put it at the end of the queue: */
  183. requeue_task_rt(rq, p);
  184. }
  185. /*
  186. * No parent/child timeslice management necessary for RT tasks,
  187. * just activate them:
  188. */
  189. static void task_new_rt(struct rq *rq, struct task_struct *p)
  190. {
  191. activate_task(rq, p, 1);
  192. }
  193. static struct sched_class rt_sched_class __read_mostly = {
  194. .enqueue_task = enqueue_task_rt,
  195. .dequeue_task = dequeue_task_rt,
  196. .yield_task = yield_task_rt,
  197. .check_preempt_curr = check_preempt_curr_rt,
  198. .pick_next_task = pick_next_task_rt,
  199. .put_prev_task = put_prev_task_rt,
  200. .load_balance = load_balance_rt,
  201. .task_tick = task_tick_rt,
  202. .task_new = task_new_rt,
  203. };