blk-mq-cpumap.c 2.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * CPU <-> hardware queue mapping helpers
  4. *
  5. * Copyright (C) 2013-2014 Jens Axboe
  6. */
  7. #include <linux/kernel.h>
  8. #include <linux/threads.h>
  9. #include <linux/module.h>
  10. #include <linux/mm.h>
  11. #include <linux/smp.h>
  12. #include <linux/cpu.h>
  13. #include <linux/blk-mq.h>
  14. #include "blk.h"
  15. #include "blk-mq.h"
  16. static int queue_index(struct blk_mq_queue_map *qmap,
  17. unsigned int nr_queues, const int q)
  18. {
  19. return qmap->queue_offset + (q % nr_queues);
  20. }
  21. static int get_first_sibling(unsigned int cpu)
  22. {
  23. unsigned int ret;
  24. ret = cpumask_first(topology_sibling_cpumask(cpu));
  25. if (ret < nr_cpu_ids)
  26. return ret;
  27. return cpu;
  28. }
  29. int blk_mq_map_queues(struct blk_mq_queue_map *qmap)
  30. {
  31. unsigned int *map = qmap->mq_map;
  32. unsigned int nr_queues = qmap->nr_queues;
  33. unsigned int cpu, first_sibling, q = 0;
  34. for_each_possible_cpu(cpu)
  35. map[cpu] = -1;
  36. /*
  37. * Spread queues among present CPUs first for minimizing
  38. * count of dead queues which are mapped by all un-present CPUs
  39. */
  40. for_each_present_cpu(cpu) {
  41. if (q >= nr_queues)
  42. break;
  43. map[cpu] = queue_index(qmap, nr_queues, q++);
  44. }
  45. for_each_possible_cpu(cpu) {
  46. if (map[cpu] != -1)
  47. continue;
  48. /*
  49. * First do sequential mapping between CPUs and queues.
  50. * In case we still have CPUs to map, and we have some number of
  51. * threads per cores then map sibling threads to the same queue
  52. * for performance optimizations.
  53. */
  54. if (q < nr_queues) {
  55. map[cpu] = queue_index(qmap, nr_queues, q++);
  56. } else {
  57. first_sibling = get_first_sibling(cpu);
  58. if (first_sibling == cpu)
  59. map[cpu] = queue_index(qmap, nr_queues, q++);
  60. else
  61. map[cpu] = map[first_sibling];
  62. }
  63. }
  64. return 0;
  65. }
  66. EXPORT_SYMBOL_GPL(blk_mq_map_queues);
  67. /**
  68. * blk_mq_hw_queue_to_node - Look up the memory node for a hardware queue index
  69. * @qmap: CPU to hardware queue map.
  70. * @index: hardware queue index.
  71. *
  72. * We have no quick way of doing reverse lookups. This is only used at
  73. * queue init time, so runtime isn't important.
  74. */
  75. int blk_mq_hw_queue_to_node(struct blk_mq_queue_map *qmap, unsigned int index)
  76. {
  77. int i;
  78. for_each_possible_cpu(i) {
  79. if (index == qmap->mq_map[i])
  80. return cpu_to_node(i);
  81. }
  82. return NUMA_NO_NODE;
  83. }