Skip to content

Commit 6d3e0c2

Browse files
author
Peter Boström
committed
Use QualityScaler for OpenH264 encoder.
BUG= R=sprang@webrtc.org Review URL: https://codereview.webrtc.org/2077393003 . Cr-Commit-Position: refs/heads/master@{#13222}
1 parent 17bde8c commit 6d3e0c2

2 files changed

Lines changed: 124 additions & 92 deletions

File tree

‎webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc‎

Lines changed: 115 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ FrameType ConvertToVideoFrameType(EVideoFrameType type) {
8181
// exclude the start codes.
8282
static void RtpFragmentize(EncodedImage* encoded_image,
8383
std::unique_ptr<uint8_t[]>* encoded_image_buffer,
84-
const VideoFrame& frame,
84+
const VideoFrameBuffer& frame_buffer,
8585
SFrameBSInfo* info,
8686
RTPFragmentationHeader* frag_header) {
8787
// Calculate minimum buffer size required to hold encoded data.
@@ -102,7 +102,8 @@ static void RtpFragmentize(EncodedImage* encoded_image,
102102
// should be more than enough to hold any encoded data of future frames of
103103
// the same size (avoiding possible future reallocation due to variations in
104104
// required size).
105-
encoded_image->_size = CalcBufferSize(kI420, frame.width(), frame.height());
105+
encoded_image->_size =
106+
CalcBufferSize(kI420, frame_buffer.width(), frame_buffer.height());
106107
if (encoded_image->_size < required_size) {
107108
// Encoded data > unencoded data. Allocate required bytes.
108109
LOG(LS_WARNING) << "Encoding produced more bytes than the original image "
@@ -198,66 +199,24 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
198199
}
199200
// else WELS_LOG_DEFAULT is used by default.
200201

202+
number_of_cores_ = number_of_cores;
201203
codec_settings_ = *codec_settings;
202204
if (codec_settings_.targetBitrate == 0)
203205
codec_settings_.targetBitrate = codec_settings_.startBitrate;
204206

205-
// Initialization parameters.
206-
// There are two ways to initialize. There is SEncParamBase (cleared with
207-
// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
208-
// which is a superset of SEncParamBase (cleared with GetDefaultParams) used
209-
// in InitializeExt.
210-
SEncParamExt init_params;
211-
openh264_encoder_->GetDefaultParams(&init_params);
212-
if (codec_settings_.mode == kRealtimeVideo) {
213-
init_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
214-
} else if (codec_settings_.mode == kScreensharing) {
215-
init_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
216-
} else {
217-
ReportError();
218-
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
219-
}
220-
init_params.iPicWidth = codec_settings_.width;
221-
init_params.iPicHeight = codec_settings_.height;
222-
// |init_params| uses bit/s, |codec_settings_| uses kbit/s.
223-
init_params.iTargetBitrate = codec_settings_.targetBitrate * 1000;
224-
init_params.iMaxBitrate = codec_settings_.maxBitrate * 1000;
225-
// Rate Control mode
226-
init_params.iRCMode = RC_BITRATE_MODE;
227-
init_params.fMaxFrameRate = static_cast<float>(codec_settings_.maxFramerate);
228-
229-
// The following parameters are extension parameters (they're in SEncParamExt,
230-
// not in SEncParamBase).
231-
init_params.bEnableFrameSkip =
232-
codec_settings_.codecSpecific.H264.frameDroppingOn;
233-
// |uiIntraPeriod| - multiple of GOP size
234-
// |keyFrameInterval| - number of frames
235-
init_params.uiIntraPeriod =
236-
codec_settings_.codecSpecific.H264.keyFrameInterval;
237-
init_params.uiMaxNalSize = 0;
238-
// Threading model: use auto.
239-
// 0: auto (dynamic imp. internal encoder)
240-
// 1: single thread (default value)
241-
// >1: number of threads
242-
init_params.iMultipleThreadIdc = NumberOfThreads(init_params.iPicWidth,
243-
init_params.iPicHeight,
244-
number_of_cores);
245-
// The base spatial layer 0 is the only one we use.
246-
init_params.sSpatialLayers[0].iVideoWidth = init_params.iPicWidth;
247-
init_params.sSpatialLayers[0].iVideoHeight = init_params.iPicHeight;
248-
init_params.sSpatialLayers[0].fFrameRate = init_params.fMaxFrameRate;
249-
init_params.sSpatialLayers[0].iSpatialBitrate = init_params.iTargetBitrate;
250-
init_params.sSpatialLayers[0].iMaxSpatialBitrate = init_params.iMaxBitrate;
251-
// Slice num according to number of threads.
252-
init_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE;
253-
207+
SEncParamExt encoder_params = CreateEncoderParams();
254208
// Initialize.
255-
if (openh264_encoder_->InitializeExt(&init_params) != 0) {
209+
if (openh264_encoder_->InitializeExt(&encoder_params) != 0) {
256210
LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
257211
Release();
258212
ReportError();
259213
return WEBRTC_VIDEO_CODEC_ERROR;
260214
}
215+
// TODO(pbos): Base init params on these values before submitting.
216+
quality_scaler_.Init(QualityScaler::kLowH264QpThreshold,
217+
QualityScaler::kBadH264QpThreshold,
218+
codec_settings_.startBitrate, codec_settings_.width,
219+
codec_settings_.height, codec_settings_.maxFramerate);
261220
int video_format = EVideoFormatType::videoFormatI420;
262221
openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT,
263222
&video_format);
@@ -276,18 +235,12 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
276235

277236
int32_t H264EncoderImpl::Release() {
278237
if (openh264_encoder_) {
279-
int uninit_ret = openh264_encoder_->Uninitialize();
280-
if (uninit_ret != 0) {
281-
LOG(LS_WARNING) << "OpenH264 encoder's Uninitialize() returned "
282-
<< "unsuccessful: " << uninit_ret;
283-
}
238+
RTC_CHECK_EQ(0, openh264_encoder_->Uninitialize());
284239
WelsDestroySVCEncoder(openh264_encoder_);
285240
openh264_encoder_ = nullptr;
286241
}
287-
if (encoded_image_._buffer != nullptr) {
288-
encoded_image_._buffer = nullptr;
289-
encoded_image_buffer_.reset();
290-
}
242+
encoded_image_._buffer = nullptr;
243+
encoded_image_buffer_.reset();
291244
return WEBRTC_VIDEO_CODEC_OK;
292245
}
293246

@@ -303,6 +256,7 @@ int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) {
303256
}
304257
codec_settings_.targetBitrate = bitrate;
305258
codec_settings_.maxFramerate = framerate;
259+
quality_scaler_.ReportFramerate(framerate);
306260

307261
SBitrateInfo target_bitrate;
308262
memset(&target_bitrate, 0, sizeof(SBitrateInfo));
@@ -316,14 +270,14 @@ int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) {
316270
return WEBRTC_VIDEO_CODEC_OK;
317271
}
318272

319-
int32_t H264EncoderImpl::Encode(
320-
const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info,
321-
const std::vector<FrameType>* frame_types) {
273+
int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
274+
const CodecSpecificInfo* codec_specific_info,
275+
const std::vector<FrameType>* frame_types) {
322276
if (!IsInitialized()) {
323277
ReportError();
324278
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
325279
}
326-
if (frame.IsZeroSize()) {
280+
if (input_frame.IsZeroSize()) {
327281
ReportError();
328282
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
329283
}
@@ -333,13 +287,20 @@ int32_t H264EncoderImpl::Encode(
333287
ReportError();
334288
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
335289
}
336-
if (frame.width() != codec_settings_.width ||
337-
frame.height() != codec_settings_.height) {
338-
LOG(LS_WARNING) << "Encoder initialized for " << codec_settings_.width
339-
<< "x" << codec_settings_.height << " but trying to encode "
340-
<< frame.width() << "x" << frame.height() << " frame.";
341-
ReportError();
342-
return WEBRTC_VIDEO_CODEC_ERR_SIZE;
290+
291+
quality_scaler_.OnEncodeFrame(input_frame.width(), input_frame.height());
292+
rtc::scoped_refptr<const VideoFrameBuffer> frame_buffer =
293+
quality_scaler_.GetScaledBuffer(input_frame.video_frame_buffer());
294+
if (frame_buffer->width() != codec_settings_.width ||
295+
frame_buffer->height() != codec_settings_.height) {
296+
LOG(LS_INFO) << "Encoder reinitialized from " << codec_settings_.width
297+
<< "x" << codec_settings_.height << " to "
298+
<< frame_buffer->width() << "x" << frame_buffer->height();
299+
codec_settings_.width = frame_buffer->width();
300+
codec_settings_.height = frame_buffer->height();
301+
SEncParamExt encoder_params = CreateEncoderParams();
302+
openh264_encoder_->SetOption(ENCODER_OPTION_SVC_ENCODE_PARAM_EXT,
303+
&encoder_params);
343304
}
344305

345306
bool force_key_frame = false;
@@ -363,16 +324,16 @@ int32_t H264EncoderImpl::Encode(
363324
// EncodeFrame input.
364325
SSourcePicture picture;
365326
memset(&picture, 0, sizeof(SSourcePicture));
366-
picture.iPicWidth = frame.width();
367-
picture.iPicHeight = frame.height();
327+
picture.iPicWidth = frame_buffer->width();
328+
picture.iPicHeight = frame_buffer->height();
368329
picture.iColorFormat = EVideoFormatType::videoFormatI420;
369-
picture.uiTimeStamp = frame.ntp_time_ms();
370-
picture.iStride[0] = frame.video_frame_buffer()->StrideY();
371-
picture.iStride[1] = frame.video_frame_buffer()->StrideU();
372-
picture.iStride[2] = frame.video_frame_buffer()->StrideV();
373-
picture.pData[0] = const_cast<uint8_t*>(frame.video_frame_buffer()->DataY());
374-
picture.pData[1] = const_cast<uint8_t*>(frame.video_frame_buffer()->DataU());
375-
picture.pData[2] = const_cast<uint8_t*>(frame.video_frame_buffer()->DataV());
330+
picture.uiTimeStamp = input_frame.ntp_time_ms();
331+
picture.iStride[0] = frame_buffer->StrideY();
332+
picture.iStride[1] = frame_buffer->StrideU();
333+
picture.iStride[2] = frame_buffer->StrideV();
334+
picture.pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
335+
picture.pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
336+
picture.pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
376337

377338
// EncodeFrame output.
378339
SFrameBSInfo info;
@@ -387,17 +348,17 @@ int32_t H264EncoderImpl::Encode(
387348
return WEBRTC_VIDEO_CODEC_ERROR;
388349
}
389350

390-
encoded_image_._encodedWidth = frame.width();
391-
encoded_image_._encodedHeight = frame.height();
392-
encoded_image_._timeStamp = frame.timestamp();
393-
encoded_image_.ntp_time_ms_ = frame.ntp_time_ms();
394-
encoded_image_.capture_time_ms_ = frame.render_time_ms();
395-
encoded_image_.rotation_ = frame.rotation();
351+
encoded_image_._encodedWidth = frame_buffer->width();
352+
encoded_image_._encodedHeight = frame_buffer->height();
353+
encoded_image_._timeStamp = input_frame.timestamp();
354+
encoded_image_.ntp_time_ms_ = input_frame.ntp_time_ms();
355+
encoded_image_.capture_time_ms_ = input_frame.render_time_ms();
356+
encoded_image_.rotation_ = input_frame.rotation();
396357
encoded_image_._frameType = ConvertToVideoFrameType(info.eFrameType);
397358

398359
// Split encoded image up into fragments. This also updates |encoded_image_|.
399360
RTPFragmentationHeader frag_header;
400-
RtpFragmentize(&encoded_image_, &encoded_image_buffer_, frame, &info,
361+
RtpFragmentize(&encoded_image_, &encoded_image_buffer_, *frame_buffer, &info,
401362
&frag_header);
402363

403364
// Encoder can skip frames to save bandwidth in which case
@@ -406,9 +367,17 @@ int32_t H264EncoderImpl::Encode(
406367
// Deliver encoded image.
407368
CodecSpecificInfo codec_specific;
408369
codec_specific.codecType = kVideoCodecH264;
409-
encoded_image_callback_->Encoded(encoded_image_,
410-
&codec_specific,
370+
encoded_image_callback_->Encoded(encoded_image_, &codec_specific,
411371
&frag_header);
372+
373+
// Parse and report QP.
374+
h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer,
375+
encoded_image_._length);
376+
int qp = -1;
377+
if (h264_bitstream_parser_.GetLastSliceQp(&qp))
378+
quality_scaler_.ReportQP(qp);
379+
} else {
380+
quality_scaler_.ReportDroppedFrame();
412381
}
413382
return WEBRTC_VIDEO_CODEC_OK;
414383
}
@@ -417,6 +386,61 @@ bool H264EncoderImpl::IsInitialized() const {
417386
return openh264_encoder_ != nullptr;
418387
}
419388

389+
// Initialization parameters.
390+
// There are two ways to initialize. There is SEncParamBase (cleared with
391+
// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
392+
// which is a superset of SEncParamBase (cleared with GetDefaultParams) used
393+
// in InitializeExt.
394+
SEncParamExt H264EncoderImpl::CreateEncoderParams() const {
395+
RTC_DCHECK(openh264_encoder_);
396+
SEncParamExt encoder_params;
397+
openh264_encoder_->GetDefaultParams(&encoder_params);
398+
if (codec_settings_.mode == kRealtimeVideo) {
399+
encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
400+
} else if (codec_settings_.mode == kScreensharing) {
401+
encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
402+
} else {
403+
RTC_NOTREACHED();
404+
}
405+
encoder_params.iPicWidth = codec_settings_.width;
406+
encoder_params.iPicHeight = codec_settings_.height;
407+
// |encoder_params| uses bit/s, |codec_settings_| uses kbit/s.
408+
encoder_params.iTargetBitrate = codec_settings_.targetBitrate * 1000;
409+
encoder_params.iMaxBitrate = codec_settings_.maxBitrate * 1000;
410+
// Rate Control mode
411+
encoder_params.iRCMode = RC_BITRATE_MODE;
412+
encoder_params.fMaxFrameRate =
413+
static_cast<float>(codec_settings_.maxFramerate);
414+
415+
// The following parameters are extension parameters (they're in SEncParamExt,
416+
// not in SEncParamBase).
417+
encoder_params.bEnableFrameSkip =
418+
codec_settings_.codecSpecific.H264.frameDroppingOn;
419+
// |uiIntraPeriod| - multiple of GOP size
420+
// |keyFrameInterval| - number of frames
421+
encoder_params.uiIntraPeriod =
422+
codec_settings_.codecSpecific.H264.keyFrameInterval;
423+
encoder_params.uiMaxNalSize = 0;
424+
// Threading model: use auto.
425+
// 0: auto (dynamic imp. internal encoder)
426+
// 1: single thread (default value)
427+
// >1: number of threads
428+
encoder_params.iMultipleThreadIdc = NumberOfThreads(
429+
encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_);
430+
// The base spatial layer 0 is the only one we use.
431+
encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth;
432+
encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight;
433+
encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate;
434+
encoder_params.sSpatialLayers[0].iSpatialBitrate =
435+
encoder_params.iTargetBitrate;
436+
encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
437+
encoder_params.iMaxBitrate;
438+
// Slice num according to number of threads.
439+
encoder_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE;
440+
441+
return encoder_params;
442+
}
443+
420444
void H264EncoderImpl::ReportInit() {
421445
if (has_reported_init_)
422446
return;
@@ -445,6 +469,7 @@ int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) {
445469
}
446470

447471
void H264EncoderImpl::OnDroppedFrame() {
472+
quality_scaler_.ReportDroppedFrame();
448473
}
449474

450475
} // namespace webrtc

‎webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@
1212
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
1313
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
1414

15-
#include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
16-
1715
#include <memory>
1816
#include <vector>
1917

18+
#include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
19+
#include "webrtc/modules/video_coding/utility/h264_bitstream_parser.h"
20+
#include "webrtc/modules/video_coding/utility/quality_scaler.h"
21+
22+
#include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
2023

2124
class ISVCEncoder;
2225

@@ -56,13 +59,17 @@ class H264EncoderImpl : public H264Encoder {
5659

5760
private:
5861
bool IsInitialized() const;
62+
SEncParamExt CreateEncoderParams() const;
5963

64+
webrtc::H264BitstreamParser h264_bitstream_parser_;
65+
QualityScaler quality_scaler_;
6066
// Reports statistics with histograms.
6167
void ReportInit();
6268
void ReportError();
6369

6470
ISVCEncoder* openh264_encoder_;
6571
VideoCodec codec_settings_;
72+
int32_t number_of_cores_;
6673

6774
EncodedImage encoded_image_;
6875
std::unique_ptr<uint8_t[]> encoded_image_buffer_;

0 commit comments

Comments
 (0)