audio_encoder_opus.cc 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/codec/audio_encoder_opus.h"
  5. #include "base/bind.h"
  6. #include "base/logging.h"
  7. #include "base/time/time.h"
  8. #include "media/base/audio_bus.h"
  9. #include "media/base/multi_channel_resampler.h"
  10. #include "third_party/opus/src/include/opus.h"
  11. namespace remoting {
  12. namespace {
  13. // Output 160 kb/s bitrate.
  14. const int kOutputBitrateBps = 160 * 1024;
  15. // Opus doesn't support 44100 sampling rate so we always resample to 48kHz.
  16. const AudioPacket::SamplingRate kOpusSamplingRate =
  17. AudioPacket::SAMPLING_RATE_48000;
  18. // Opus supports frame sizes of 2.5, 5, 10, 20, 40 and 60 ms. We use 20 ms
  19. // frames to balance latency and efficiency.
  20. const int kFrameSizeMs = 20;
  21. // Number of samples per frame when using default sampling rate.
  22. const int kFrameSamples =
  23. kOpusSamplingRate * kFrameSizeMs / base::Time::kMillisecondsPerSecond;
  24. const AudioPacket::BytesPerSample kBytesPerSample =
  25. AudioPacket::BYTES_PER_SAMPLE_2;
  26. bool IsSupportedSampleRate(int rate) {
  27. return rate == 44100 || rate == 48000;
  28. }
  29. } // namespace
  30. AudioEncoderOpus::AudioEncoderOpus()
  31. : sampling_rate_(0),
  32. channels_(AudioPacket::CHANNELS_STEREO),
  33. encoder_(nullptr),
  34. frame_size_(0),
  35. resampling_data_(nullptr),
  36. resampling_data_size_(0),
  37. resampling_data_pos_(0) {
  38. }
  39. AudioEncoderOpus::~AudioEncoderOpus() {
  40. DestroyEncoder();
  41. }
  42. void AudioEncoderOpus::InitEncoder() {
  43. DCHECK(!encoder_);
  44. int error;
  45. encoder_ = opus_encoder_create(kOpusSamplingRate, channels_,
  46. OPUS_APPLICATION_AUDIO, &error);
  47. if (!encoder_) {
  48. LOG(ERROR) << "Failed to create OPUS encoder. Error code: " << error;
  49. return;
  50. }
  51. opus_encoder_ctl(encoder_.get(), OPUS_SET_BITRATE(kOutputBitrateBps));
  52. frame_size_ = sampling_rate_ * kFrameSizeMs /
  53. base::Time::kMillisecondsPerSecond;
  54. if (sampling_rate_ != kOpusSamplingRate) {
  55. resample_buffer_.reset(
  56. new char[kFrameSamples * kBytesPerSample * channels_]);
  57. // TODO(sergeyu): Figure out the right buffer size to use per packet instead
  58. // of using media::SincResampler::kDefaultRequestSize.
  59. resampler_ = std::make_unique<media::MultiChannelResampler>(
  60. channels_, sampling_rate_ / double{kOpusSamplingRate},
  61. media::SincResampler::kDefaultRequestSize,
  62. base::BindRepeating(&AudioEncoderOpus::FetchBytesToResample,
  63. base::Unretained(this)));
  64. resampler_bus_ = media::AudioBus::Create(channels_, kFrameSamples);
  65. }
  66. // Drop leftover data because it's for different sampling rate.
  67. leftover_samples_ = 0;
  68. leftover_buffer_size_ =
  69. frame_size_ + media::SincResampler::kDefaultRequestSize;
  70. leftover_buffer_.reset(new int16_t[leftover_buffer_size_ * channels_]);
  71. }
  72. void AudioEncoderOpus::DestroyEncoder() {
  73. if (encoder_) {
  74. opus_encoder_destroy(encoder_);
  75. encoder_ = nullptr;
  76. }
  77. resampler_.reset();
  78. }
  79. bool AudioEncoderOpus::ResetForPacket(AudioPacket* packet) {
  80. if (packet->channels() != channels_ ||
  81. packet->sampling_rate() != sampling_rate_) {
  82. DestroyEncoder();
  83. channels_ = packet->channels();
  84. sampling_rate_ = packet->sampling_rate();
  85. if (channels_ <= 0 || channels_ > 2 ||
  86. !IsSupportedSampleRate(sampling_rate_)) {
  87. LOG(WARNING) << "Unsupported OPUS parameters: "
  88. << channels_ << " channels with "
  89. << sampling_rate_ << " samples per second.";
  90. return false;
  91. }
  92. InitEncoder();
  93. }
  94. return encoder_ != nullptr;
  95. }
  96. void AudioEncoderOpus::FetchBytesToResample(int resampler_frame_delay,
  97. media::AudioBus* audio_bus) {
  98. DCHECK(resampling_data_);
  99. int samples_left = (resampling_data_size_ - resampling_data_pos_) /
  100. kBytesPerSample / channels_;
  101. DCHECK_LE(audio_bus->frames(), samples_left);
  102. static_assert(kBytesPerSample == 2, "FromInterleaved expects 2 bytes.");
  103. audio_bus->FromInterleaved<media::SignedInt16SampleTypeTraits>(
  104. reinterpret_cast<const int16_t*>(resampling_data_ + resampling_data_pos_),
  105. audio_bus->frames());
  106. resampling_data_pos_ += audio_bus->frames() * kBytesPerSample * channels_;
  107. DCHECK_LE(resampling_data_pos_, static_cast<int>(resampling_data_size_));
  108. }
  109. int AudioEncoderOpus::GetBitrate() {
  110. return kOutputBitrateBps;
  111. }
  112. std::unique_ptr<AudioPacket> AudioEncoderOpus::Encode(
  113. std::unique_ptr<AudioPacket> packet) {
  114. DCHECK_EQ(AudioPacket::ENCODING_RAW, packet->encoding());
  115. DCHECK_EQ(1, packet->data_size());
  116. DCHECK_EQ(kBytesPerSample, packet->bytes_per_sample());
  117. if (!ResetForPacket(packet.get())) {
  118. LOG(ERROR) << "Encoder initialization failed";
  119. return nullptr;
  120. }
  121. int samples_in_packet = packet->data(0).size() / kBytesPerSample / channels_;
  122. const int16_t* next_sample =
  123. reinterpret_cast<const int16_t*>(packet->data(0).data());
  124. // Create a new packet of encoded data.
  125. std::unique_ptr<AudioPacket> encoded_packet(new AudioPacket());
  126. encoded_packet->set_encoding(AudioPacket::ENCODING_OPUS);
  127. encoded_packet->set_sampling_rate(kOpusSamplingRate);
  128. encoded_packet->set_channels(channels_);
  129. int prefetch_samples =
  130. resampler_.get() ? media::SincResampler::kDefaultRequestSize : 0;
  131. int samples_wanted = frame_size_ + prefetch_samples;
  132. while (leftover_samples_ + samples_in_packet >= samples_wanted) {
  133. const int16_t* pcm_buffer = nullptr;
  134. // Combine the packet with the leftover samples, if any.
  135. if (leftover_samples_ > 0) {
  136. pcm_buffer = leftover_buffer_.get();
  137. int samples_to_copy = samples_wanted - leftover_samples_;
  138. memcpy(leftover_buffer_.get() + leftover_samples_ * channels_,
  139. next_sample, samples_to_copy * kBytesPerSample * channels_);
  140. } else {
  141. pcm_buffer = next_sample;
  142. }
  143. // Resample data if necessary.
  144. int samples_consumed = 0;
  145. if (resampler_.get()) {
  146. resampling_data_ = reinterpret_cast<const char*>(pcm_buffer);
  147. resampling_data_pos_ = 0;
  148. resampling_data_size_ = samples_wanted * channels_ * kBytesPerSample;
  149. resampler_->Resample(kFrameSamples, resampler_bus_.get());
  150. resampling_data_ = nullptr;
  151. samples_consumed = resampling_data_pos_ / channels_ / kBytesPerSample;
  152. static_assert(kBytesPerSample == 2, "ToInterleaved expects 2 bytes.");
  153. resampler_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>(
  154. kFrameSamples, reinterpret_cast<int16_t*>(resample_buffer_.get()));
  155. pcm_buffer = reinterpret_cast<int16_t*>(resample_buffer_.get());
  156. } else {
  157. samples_consumed = frame_size_;
  158. }
  159. // Initialize output buffer.
  160. std::string* data = encoded_packet->add_data();
  161. data->resize(kFrameSamples * kBytesPerSample * channels_);
  162. // Encode.
  163. unsigned char* buffer = reinterpret_cast<unsigned char*>(std::data(*data));
  164. int result = opus_encode(encoder_, pcm_buffer, kFrameSamples,
  165. buffer, data->length());
  166. if (result < 0) {
  167. LOG(ERROR) << "opus_encode() failed with error code: " << result;
  168. return nullptr;
  169. }
  170. DCHECK_LE(result, static_cast<int>(data->length()));
  171. data->resize(result);
  172. // Cleanup leftover buffer.
  173. if (samples_consumed >= leftover_samples_) {
  174. samples_consumed -= leftover_samples_;
  175. leftover_samples_ = 0;
  176. next_sample += samples_consumed * channels_;
  177. samples_in_packet -= samples_consumed;
  178. } else {
  179. leftover_samples_ -= samples_consumed;
  180. memmove(leftover_buffer_.get(),
  181. leftover_buffer_.get() + samples_consumed * channels_,
  182. leftover_samples_ * channels_ * kBytesPerSample);
  183. }
  184. }
  185. // Store the leftover samples.
  186. if (samples_in_packet > 0) {
  187. DCHECK_LE(leftover_samples_ + samples_in_packet, leftover_buffer_size_);
  188. memmove(leftover_buffer_.get() + leftover_samples_ * channels_,
  189. next_sample, samples_in_packet * kBytesPerSample * channels_);
  190. leftover_samples_ += samples_in_packet;
  191. }
  192. // Return nullptr if there's nothing in the packet.
  193. if (encoded_packet->data_size() == 0)
  194. return nullptr;
  195. return encoded_packet;
  196. }
  197. } // namespace remoting