0009-Fix-the-sync-issue-between-the-Preview.Show-and-the-.patch 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. From e9e9555cac3c11c70277824d82d5b3fbd5e4afd2 Mon Sep 17 00:00:00 2001
  2. From: "zejian.su" <zejian.su@starfivetech.com>
  3. Date: Fri, 1 Dec 2023 16:30:27 +0800
  4. Subject: [PATCH 2/2] Fix the sync issue between the Preview.Show() and the
  5. QWidget.update() 1. Add a mutex between the Preview.Show() and the
  6. QWidget.update(). 2. Speed up the Preview.Show() -- change the floating point
  7. to integer, merge some plus and multiplying operation and remove some data
  8. copy.
  9. Signed-off-by: zejian.su <zejian.su@starfivetech.com>
  10. ---
  11. preview/qt_preview.cpp | 183 +++++++++++++++++++++++++----------------
  12. 1 file changed, 113 insertions(+), 70 deletions(-)
  13. diff --git a/preview/qt_preview.cpp b/preview/qt_preview.cpp
  14. index 8196a8e..ef3c8a4 100644
  15. --- a/preview/qt_preview.cpp
  16. +++ b/preview/qt_preview.cpp
  17. @@ -41,16 +41,47 @@ class MyWidget : public QWidget
  18. public:
  19. MyWidget(QWidget *parent, int w, int h) : QWidget(parent), size(w, h)
  20. {
  21. - image = QImage(size, QImage::Format_RGB888);
  22. - image.fill(0);
  23. + for(int i = 0; i < 2; i++) {
  24. + buffers_[i].image = QImage(size, QImage::Format_RGB888);
  25. + buffers_[i].image.fill(0);
  26. + availableBuffers_.push_back(&buffers_[i]);
  27. + }
  28. }
  29. QSize size;
  30. - QImage image;
  31. +
  32. + struct ImageBuffer {
  33. + QImage image;
  34. + uint8_t frameCounter;
  35. +
  36. + ImageBuffer() : frameCounter(0) {};
  37. + } buffers_[2];
  38. +
  39. + std::list<ImageBuffer *> freeBuffers_;
  40. + std::list<ImageBuffer *> availableBuffers_;
  41. +
  42. + std::mutex buffers_available_mutex_;
  43. + std::mutex buffers_free_mutex_;
  44. protected:
  45. void paintEvent(QPaintEvent *) override
  46. {
  47. QPainter painter(this);
  48. - painter.drawImage(rect(), image, image.rect());
  49. + {
  50. + ImageBuffer *buffer = nullptr;
  51. + {
  52. + std::lock_guard<std::mutex> lock(buffers_available_mutex_);
  53. + if(!availableBuffers_.size())
  54. + return;
  55. + buffer = availableBuffers_.back();
  56. + availableBuffers_.pop_back();
  57. + }
  58. +
  59. + painter.drawImage(rect(), buffer->image, buffer->image.rect());
  60. +
  61. + {
  62. + std::lock_guard<std::mutex> lock(buffers_free_mutex_);
  63. + freeBuffers_.push_back(buffer);
  64. + }
  65. + }
  66. }
  67. QSize sizeHint() const override { return size; }
  68. };
  69. @@ -58,7 +89,7 @@ protected:
  70. class QtPreview : public Preview
  71. {
  72. public:
  73. - QtPreview(Options const *options) : Preview(options), frame_counter_(0)
  74. + QtPreview(Options const *options) : Preview(options)
  75. {
  76. window_width_ = options->preview_width;
  77. window_height_ = options->preview_height;
  78. @@ -67,6 +98,8 @@ public:
  79. // This preview window is expensive, so make it small by default.
  80. if (window_width_ == 0 || window_height_ == 0)
  81. window_width_ = 512, window_height_ = 384;
  82. +
  83. + frameCounter_ = 0;
  84. // As a hint, reserve twice the binned width for our widest current camera (V3)
  85. tmp_stripe_.reserve(4608);
  86. thread_ = std::thread(&QtPreview::threadFunc, this, options);
  87. @@ -83,100 +116,105 @@ public:
  88. void SetInfoText(const std::string &text) override { main_window_->setWindowTitle(QString::fromStdString(text)); }
  89. virtual void Show(int fd, libcamera::Span<uint8_t> span, StreamInfo const &info) override
  90. {
  91. - if((frame_counter_++) & 1) {
  92. - // Return the buffer to the camera system.
  93. - done_callback_(fd);
  94. - return;
  95. + MyWidget::ImageBuffer *buffer = nullptr;
  96. + {
  97. + std::lock_guard<std::mutex> lock(pane_->buffers_free_mutex_);
  98. + if(pane_->freeBuffers_.size() > 0) {
  99. + buffer = pane_->freeBuffers_.front();
  100. + pane_->freeBuffers_.pop_front();
  101. + }
  102. + }
  103. +
  104. + if(!buffer) {
  105. + std::lock_guard<std::mutex> lock(pane_->buffers_available_mutex_);
  106. + if(!pane_->availableBuffers_.size()) {
  107. + done_callback_(fd);
  108. + return;
  109. + }
  110. + buffer = pane_->availableBuffers_.front();
  111. + pane_->availableBuffers_.pop_front();
  112. }
  113. -
  114. // Quick and simple nearest-neighbour-ish resampling is used here.
  115. // We further share U,V samples between adjacent output pixel pairs
  116. // (even when downscaling) to speed up the conversion.
  117. unsigned x_step = (info.width << 16) / window_width_;
  118. unsigned y_step = (info.height << 16) / window_height_;
  119. - // Choose the right matrix to convert YUV back to RGB.
  120. - static const float YUV2RGB[3][9] = {
  121. - { 1.0, 0.0, 1.402, 1.0, -0.344, -0.714, 1.0, 1.772, 0.0 }, // JPEG
  122. - { 1.164, 0.0, 1.596, 1.164, -0.392, -0.813, 1.164, 2.017, 0.0 }, // SMPTE170M
  123. - { 1.164, 0.0, 1.793, 1.164, -0.213, -0.533, 1.164, 2.112, 0.0 }, // Rec709
  124. + static const uint32_t YUV2RGB[3][9] = {
  125. + { 128, 0, 179, 128, 44, 91, 128, 227, 0 }, // JPEG
  126. + { 149, 0, 204, 149, 50, 104, 149, 258, 0 }, // SMPTE170M
  127. + { 149, 0, 230, 149, 27, 68, 149, 270, 0 }, // Rec709
  128. };
  129. - int offsetY;
  130. - float coeffY, coeffVR, coeffUG, coeffVG, coeffUB;
  131. - if (info.colour_space == libcamera::ColorSpace::Smpte170m)
  132. + static const int RGBOFFSET[3][3] = {
  133. + {24960, -15232, 31104}, {28496, -17328, 35408}, {31824, -9776, 36944}
  134. + };
  135. +
  136. + uint32_t coeffY, coeffVR, coeffUG, coeffVG, coeffUB;
  137. + const int * rgbOffset = nullptr;
  138. + if(info.colour_space == libcamera::ColorSpace::Smpte170m)
  139. {
  140. - offsetY = 16;
  141. coeffY = YUV2RGB[1][0];
  142. coeffVR = YUV2RGB[1][2];
  143. coeffUG = YUV2RGB[1][4];
  144. coeffVG = YUV2RGB[1][5];
  145. coeffUB = YUV2RGB[1][7];
  146. - }
  147. - else if (info.colour_space == libcamera::ColorSpace::Rec709)
  148. + rgbOffset = RGBOFFSET[1];
  149. + } else if(info.colour_space == libcamera::ColorSpace::Rec709)
  150. {
  151. - offsetY = 16;
  152. coeffY = YUV2RGB[2][0];
  153. coeffVR = YUV2RGB[2][2];
  154. coeffUG = YUV2RGB[2][4];
  155. coeffVG = YUV2RGB[2][5];
  156. coeffUB = YUV2RGB[2][7];
  157. - }
  158. - else
  159. + rgbOffset = RGBOFFSET[2];
  160. + } else
  161. {
  162. - offsetY = 0;
  163. coeffY = YUV2RGB[0][0];
  164. coeffVR = YUV2RGB[0][2];
  165. coeffUG = YUV2RGB[0][4];
  166. coeffVG = YUV2RGB[0][5];
  167. coeffUB = YUV2RGB[0][7];
  168. - if (info.colour_space != libcamera::ColorSpace::Sycc)
  169. - LOG(1, "QtPreview: unexpected colour space " << libcamera::ColorSpace::toString(info.colour_space));
  170. + rgbOffset = RGBOFFSET[0];
  171. }
  172. - // Because the source buffer is uncached, and we want to read it a byte at a time,
  173. - // take a copy of each row used. This is a speedup provided memcpy() is vectorized.
  174. - tmp_stripe_.resize(2 * info.stride);
  175. - uint8_t const *Y_start = span.data();
  176. - uint8_t const *UV_start = Y_start + info.height * info.stride;
  177. - uint8_t *Y_row = &tmp_stripe_[0];
  178. - uint8_t *U_row = Y_row + info.stride;
  179. - uint8_t *V_row = U_row + (info.stride >> 1);
  180. -
  181. - // Possibly this should be locked in case a repaint is happening? In practice the risk
  182. - // is only that there might be some tearing, so I don't think we worry. We could speed
  183. - // it up by getting the ISP to supply RGB, but I'm not sure I want to handle that extra
  184. - // possibility in our main application code, so we'll put up with the slow conversion.
  185. - for (unsigned int y = 0; y < window_height_; y++)
  186. + uint8_t const * Y_start = span.data();
  187. + uint8_t const * UV_start = Y_start + info.height * info.stride;
  188. + int src_ypos = y_step >> 1;
  189. + uint32_t Y2 = 0;
  190. + uint32_t U2 = coeffUG | (coeffUB << 16);
  191. + uint32_t V2 = coeffVR | (coeffVG << 16);
  192. + uint16_t * y2 = (uint16_t *)&Y2;
  193. + uint32_t U;
  194. + uint32_t V;
  195. + uint16_t * u2 = (uint16_t *)&U;
  196. + uint16_t * v2 = (uint16_t *)&V;
  197. +
  198. + for(unsigned int y = 0; y < window_height_; y++, src_ypos += y_step)
  199. {
  200. - unsigned row = (y * y_step) >> 16;
  201. - uint8_t *dest = pane_->image.scanLine(y);
  202. - unsigned x_pos = x_step >> 1;
  203. -
  204. - memcpy(Y_row, Y_start + row * info.stride, info.stride);
  205. - //memcpy(U_row, Y_start + ((4 * info.height + row) >> 1) * (info.stride >> 1), info.stride >> 1);
  206. - //memcpy(V_row, Y_start + ((5 * info.height + row) >> 1) * (info.stride >> 1), info.stride >> 1);
  207. - uint8_t const *cur_uv = UV_start + (row >> 1) * info.width;
  208. - for(unsigned int uv_idx = 0; uv_idx < info.width >> 1; uv_idx++, cur_uv += 2)
  209. - U_row[uv_idx] = cur_uv[0], V_row[uv_idx] = cur_uv[1];
  210. -
  211. - for (unsigned int x = 0; x < window_width_; x += 2)
  212. + const uint8_t * src_y = Y_start + (src_ypos >> 16) * info.width;
  213. + const uint8_t * src_uv = UV_start + (src_ypos >> 17) * info.width;
  214. + uint8_t * dest = buffer->image.scanLine(y);
  215. + uint32_t x_pos = x_step >> 1;
  216. +
  217. + for(unsigned int x = 0; x < window_width_; x += 2)
  218. {
  219. - int Y0 = Y_row[x_pos >> 16];
  220. + y2[0] = src_y[x_pos >> 16];
  221. x_pos += x_step;
  222. - int Y1 = Y_row[x_pos >> 16];
  223. - int U = U_row[x_pos >> 17];
  224. - int V = V_row[x_pos >> 17];
  225. + y2[1] = src_y[x_pos >> 16];
  226. + U = src_uv[(x_pos >> 16) & 0xfffffffe];
  227. + V = src_uv[(x_pos >> 16) | 1];
  228. x_pos += x_step;
  229. - Y0 -= offsetY;
  230. - Y1 -= offsetY;
  231. - U -= 128;
  232. - V -= 128;
  233. - int R0 = coeffY * Y0 + coeffVR * V;
  234. - int G0 = coeffY * Y0 + coeffUG * U + coeffVG * V;
  235. - int B0 = coeffY * Y0 + coeffUB * U;
  236. - int R1 = coeffY * Y1 + coeffVR * V;
  237. - int G1 = coeffY * Y1 + coeffUG * U + coeffVG * V;
  238. - int B1 = coeffY * Y1 + coeffUB * U;
  239. +
  240. + Y2 *= coeffY;
  241. + U *= U2;
  242. + V *= V2;
  243. +
  244. + int R0 = ((int)y2[0] + (int)v2[0] - rgbOffset[0]) >> 7;
  245. + int G0 = ((int)y2[0] - (int)u2[0] - (int)v2[1] - rgbOffset[1]) >> 7;
  246. + int B0 = ((int)y2[0] + (int)u2[1] - rgbOffset[2]) >> 7;
  247. + int R1 = ((int)y2[1] + (int)v2[0] - rgbOffset[0]) >> 7;
  248. + int G1 = ((int)y2[1] - (int)u2[0] - (int)v2[1] - rgbOffset[1]) >> 7;
  249. + int B1 = ((int)y2[1] + (int)u2[1] - rgbOffset[2]) >> 7;
  250. *(dest++) = std::clamp(R0, 0, 255);
  251. *(dest++) = std::clamp(G0, 0, 255);
  252. *(dest++) = std::clamp(B0, 0, 255);
  253. @@ -186,6 +224,12 @@ public:
  254. }
  255. }
  256. + {
  257. + std::lock_guard<std::mutex> lock(pane_->buffers_available_mutex_);
  258. + buffer->frameCounter = ++frameCounter_;
  259. + pane_->availableBuffers_.push_back(buffer);
  260. + }
  261. +
  262. pane_->update();
  263. // Return the buffer to the camera system.
  264. @@ -193,7 +237,7 @@ public:
  265. }
  266. // Reset the preview window, clearing the current buffers and being ready to
  267. // show new ones.
  268. - void Reset() override {frame_counter_ = 0;}
  269. + void Reset() override {}
  270. // Check if preview window has been shut down.
  271. bool Quit() override { return main_window_->quit; }
  272. // There is no particular limit to image sizes, though large images will be very slow.
  273. @@ -228,8 +272,7 @@ private:
  274. std::mutex mutex_;
  275. std::condition_variable cond_var_;
  276. std::vector<uint8_t> tmp_stripe_;
  277. -
  278. - unsigned int frame_counter_;
  279. + uint8_t frameCounter_;
  280. };
  281. Preview *make_qt_preview(Options const *options)
  282. --
  283. 2.34.1