From 94317d1a390f297d182b09f8cddd70c8f6d5c198 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 7 Aug 2015 21:40:48 -0400 Subject: [PATCH 01/23] added graph display of instantly required bandwidth for VOIP, in preparation to chosing new video codec. GUI layout needs to be sorted. --- plugins/VOIP/gui/AudioInputConfig.cpp | 66 ++++++++++++++++++++++- plugins/VOIP/gui/AudioInputConfig.h | 20 ++++++- plugins/VOIP/gui/AudioInputConfig.ui | 76 +++++++++++++++++---------- plugins/VOIP/gui/QVideoDevice.cpp | 5 +- plugins/VOIP/interface/rsVOIP.h | 2 + plugins/VOIP/services/p3VOIP.cc | 8 +++ 6 files changed, 147 insertions(+), 30 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 5c7553ee8..e23083200 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -39,6 +39,7 @@ //#include "NetworkConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" +#include "gui/common/RSGraphWidget.h" #include #define iroundf(x) ( static_cast(x) ) @@ -51,6 +52,59 @@ void AudioInputDialog::showEvent(QShowEvent *) { qtTick->start(20); }*/ +class voipGraphSource: public RSGraphSource +{ +public: + voipGraphSource() {} + + void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; } + + virtual QString displayName(int) const { return tr("Required bandwidth") ;} + + virtual QString displayValue(float v) const + { + if(v < 1000) + return QString::number(v,10,2) + " B/s" ; + else if(v < 1000*1024) + return QString::number(v/1024,10,2) + " KB/s" ; + else + return QString::number(v/(1024*1024),10,2) + " MB/s" ; + } + + virtual void getValues(std::map& vals) const + { + RsVOIPDataChunk chunk ; + uint32_t total_size = 0 ; + vals.clear() ; + + while(video_input && video_input->getNextEncodedPacket(chunk)) + { + total_size += chunk.size ; + chunk.clear() ; + } + + vals[std::string("bw")] = (float)total_size ; + } + +private: + QVideoInputDevice *video_input ; +}; + +void voipGraph::setVoipSource(voipGraphSource *gs) +{ +_src = gs ; +RSGraphWidget::setSource(gs) ; +} + +voipGraph::voipGraph(QWidget *parent) + : RSGraphWidget(parent) +{ + setFlags(RSGraphWidget::RSGRAPH_FLAGS_SHOW_LEGEND) ; + setFlags(RSGraphWidget::RSGRAPH_FLAGS_PAINT_STYLE_PLAIN) ; + + _src = NULL ; +} + /** Constructor */ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) @@ -71,11 +125,21 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) // videoInput = new QVideoInputDevice(this) ; videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(NULL) ; + videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); + + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; } AudioInputConfig::~AudioInputConfig() { + graph_source->stop() ; + graph_source->setVideoInput(NULL) ; + std::cerr << "Deleting audioInputConfig object" << std::endl; if(videoInput != NULL) { diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index 1b64085a3..cc0fd15c3 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -35,10 +35,27 @@ #include "retroshare-gui/configpage.h" -#include "ui_AudioInputConfig.h" #include "SpeexProcessor.h" #include "VideoProcessor.h" #include "AudioStats.h" +#include "gui/common/RSGraphWidget.h" + +class voipGraphSource ; + +class voipGraph: public RSGraphWidget +{ +public: + voipGraph(QWidget *parent) ; + + voipGraphSource *voipSource() const { return _src ; } + + void setVoipSource(voipGraphSource *gs) ; + +private: + voipGraphSource *_src ; +}; + +#include "ui_AudioInputConfig.h" class AudioInputConfig : public ConfigPage { @@ -54,6 +71,7 @@ class AudioInputConfig : public ConfigPage QVideoInputDevice *videoInput ; bool loaded; + voipGraphSource *graph_source ; protected: QTimer *qtTick; diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 6aa071b4a..8037dea93 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -6,8 +6,8 @@ 0 0 - 501 - 406 + 1155 + 713 @@ -348,33 +348,49 @@ - - - - Video Processing - - - - - - - 170 - 128 - - - - QFrame::StyledPanel - - - QFrame::Raised - - - - - - + + + + Video Processing + + + + + + + 170 + 128 + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + + + 0 + 0 + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + @@ -397,6 +413,12 @@
gui/QVideoDevice.h
1 + + voipGraph + QFrame +
gui/AudioInputConfig.h
+ 1 + qcbTransmit diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index eed14b58b..eaec96f71 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -88,7 +88,10 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { - return _video_encoder->nextPacket(chunk) ; + if(_video_encoder) + return _video_encoder->nextPacket(chunk) ; + else + return false ; } QVideoInputDevice::~QVideoInputDevice() diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 18f258769..7269610b4 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -56,6 +56,8 @@ struct RsVOIPDataChunk void *data ; // create/delete using malloc/free. uint32_t size ; RsVOIPDataType type ; // video or audio + + void clear() ; }; class RsVOIP diff --git a/plugins/VOIP/services/p3VOIP.cc b/plugins/VOIP/services/p3VOIP.cc index d7377077c..fb164bd7c 100644 --- a/plugins/VOIP/services/p3VOIP.cc +++ b/plugins/VOIP/services/p3VOIP.cc @@ -147,6 +147,14 @@ RsServiceInfo p3VOIP::getServiceInfo() TURTLE_MIN_MINOR_VERSION); } +void RsVOIPDataChunk::clear() +{ + + if(data) + free(data) ; + data=NULL; + size=0 ; +} int p3VOIP::tick() { #ifdef DEBUG_VOIP From 2107a1c858cca4565435d876056ec5978f6c5999 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 8 Aug 2015 18:46:49 -0400 Subject: [PATCH 02/23] fixed up image display layout in VOIP config tab --- plugins/VOIP/gui/QVideoDevice.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index eaec96f71..67c79ee5b 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -113,6 +113,7 @@ void QVideoOutputDevice::showFrameOff() void QVideoOutputDevice::showFrame(const QImage& img) { - setPixmap(QPixmap::fromImage(img).scaled(minimumSize(),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; + std::cerr << "img.size = " << img.width() << " x " << img.height() << std::endl; + setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*640/480,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; } From c0614e70ac235724fe74085197e602f4978bfa47 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 9 Aug 2015 18:09:17 -0400 Subject: [PATCH 03/23] added preview option to show decoded stream in Audio config --- plugins/VOIP/gui/AudioInputConfig.cpp | 45 ++++++++++++----- plugins/VOIP/gui/AudioInputConfig.h | 2 + plugins/VOIP/gui/AudioInputConfig.ui | 10 ++++ plugins/VOIP/gui/QVideoDevice.cpp | 61 ++++++++++++++-------- plugins/VOIP/gui/QVideoDevice.h | 10 ++++ plugins/VOIP/gui/VideoProcessor.cpp | 73 ++++++++++++++++++++------- plugins/VOIP/gui/VideoProcessor.h | 53 +++++++++++++++---- plugins/VOIP/interface/rsVOIP.h | 4 +- 8 files changed, 195 insertions(+), 63 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index e23083200..d9931a2af 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -55,9 +55,9 @@ void AudioInputDialog::showEvent(QShowEvent *) { class voipGraphSource: public RSGraphSource { public: - voipGraphSource() {} + voipGraphSource() : video_input(NULL) {} - void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; } + void setVideoInput(const QVideoInputDevice *vid) { video_input = vid ; } virtual QString displayName(int) const { return tr("Required bandwidth") ;} @@ -73,21 +73,14 @@ public: virtual void getValues(std::map& vals) const { - RsVOIPDataChunk chunk ; - uint32_t total_size = 0 ; vals.clear() ; - while(video_input && video_input->getNextEncodedPacket(chunk)) - { - total_size += chunk.size ; - chunk.clear() ; - } - - vals[std::string("bw")] = (float)total_size ; + if(video_input) + vals[std::string("bw")] = video_input->currentBandwidth() ; } private: - QVideoInputDevice *video_input ; + const QVideoInputDevice *video_input ; }; void voipGraph::setVoipSource(voipGraphSource *gs) @@ -127,12 +120,31 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) videoInput->setEchoVideoTarget(ui.videoDisplay) ; videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + videoDecoder = new JPEGVideoDecoder; + videoDecoder->setDisplayTarget(NULL) ; + graph_source = new voipGraphSource ; ui.voipBwGraph->setSource(graph_source); graph_source->setVideoInput(videoInput) ; graph_source->setCollectionTimeLimit(1000*300) ; graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; +} + +void AudioInputConfig::togglePreview(bool b) +{ + if(b) + { + videoInput->setEchoVideoTarget(NULL) ; + videoDecoder->setDisplayTarget(ui.videoDisplay) ; + } + else + { + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + videoDecoder->setDisplayTarget(NULL) ; + } } AudioInputConfig::~AudioInputConfig() @@ -352,6 +364,15 @@ void AudioInputConfig::on_Tick_timeout() { abSpeech->iValue = iroundf(inputAudioProcessor->dVoiceAcivityLevel * 32767.0f + 0.5f); abSpeech->update(); + + // also transmit encoded video + RsVOIPDataChunk chunk ; + + while(videoInput->getNextEncodedPacket(chunk)) + { + videoDecoder->receiveEncodedData(static_cast(chunk.data),chunk.size) ; + chunk.clear() ; + } } void AudioInputConfig::emptyBuffer() { diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index cc0fd15c3..faaadb63e 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -69,6 +69,7 @@ class AudioInputConfig : public ConfigPage //VideoDecoder *videoDecoder ; //VideoEncoder *videoEncoder ; QVideoInputDevice *videoInput ; + VideoDecoder *videoDecoder ; bool loaded; voipGraphSource *graph_source ; @@ -96,6 +97,7 @@ class AudioInputConfig : public ConfigPage private slots: void loadSettings(); void emptyBuffer(); + void togglePreview(bool) ; void on_qsTransmitHold_valueChanged(int v); void on_qsAmp_valueChanged(int v); diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 8037dea93..7b77fb3a9 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -391,6 +391,16 @@ + + + + <html><head/><body><p>Display encoded (and then decoded) frame, to check the codec's quality. If not selected, the image above only shows the frame that is grabbed from your camera.</p></body></html> + + + preview + + + diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 67c79ee5b..3a7ee029e 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -13,6 +13,9 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _capture_device = NULL ; _video_encoder = NULL ; _echo_output_device = NULL ; + _estimated_bw = 0 ; + _total_encoded_size = 0 ; + _last_bw_estimate_TS = time(NULL) ; } void QVideoInputDevice::stop() @@ -54,36 +57,52 @@ void QVideoInputDevice::start() void QVideoInputDevice::grabFrame() { - IplImage *img=cvQueryFrame(_capture_device); + IplImage *img=cvQueryFrame(_capture_device); - if(img == NULL) - { - std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; - return ; - } - // get the image data + if(img == NULL) + { + std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; + return ; + } + // get the image data - if(img->nChannels != 3) - { - std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; - return ; - } + if(img->nChannels != 3) + { + std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; + return ; + } // convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer cv::Mat img_rgb; cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB); - static const int _encoded_width = 128 ; - static const int _encoded_height = 128 ; + QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + if(_video_encoder != NULL) + { + uint32_t encoded_size ; - if(_video_encoder != NULL) - { - _video_encoder->addImage(image) ; - emit networkPacketReady() ; - } - if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ; + _video_encoder->addImage(image,0,encoded_size) ; + + std::cerr << "Encoded size = " << encoded_size << std::endl; + _total_encoded_size += encoded_size ; + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_TS) + { + _estimated_bw = uint32_t(0.75*_estimated_bw + 0.25 * (_total_encoded_size / (float)(now - _last_bw_estimate_TS))) ; + + _total_encoded_size = 0 ; + _last_bw_estimate_TS = now ; + + std::cerr << "new bw estimate: " << _estimated_bw << std::endl; + } + + emit networkPacketReady() ; + } + if(_echo_output_device != NULL) + _echo_output_device->showFrame(image) ; } bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 7cb62ca0c..513932487 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -42,6 +42,12 @@ class QVideoInputDevice: public QObject // bool getNextEncodedPacket(RsVOIPDataChunk&) ; + // gets the estimated current bandwidth required to transmit the encoded data, in B/s + // + uint32_t currentBandwidth() const { return _estimated_bw ; } + + // control + void start() ; void stop() ; @@ -59,5 +65,9 @@ class QVideoInputDevice: public QObject QVideoOutputDevice *_echo_output_device ; std::list _out_queue ; + + uint32_t _estimated_bw ; + time_t _last_bw_estimate_TS; + uint32_t _total_encoded_size ; }; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 27343c412..c9add1b26 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,9 +12,15 @@ VideoDecoder::VideoDecoder() _output_device = NULL ; } -bool VideoEncoder::addImage(const QImage& img) +VideoEncoder::VideoEncoder() + :_frame_size(128,128) { - encodeData(img) ; +} + +bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +{ + std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; + encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; return true ; } @@ -32,6 +38,7 @@ bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk) void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) { + if(_output_device) _output_device->showFrame(decodeData(data,size)) ; } @@ -50,24 +57,54 @@ QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) { - std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; + std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; } - -void JPEGVideoEncoder::encodeData(const QImage& image) +void VideoEncoder::setInternalFrameSize(QSize s) { - QByteArray qb ; - - QBuffer buffer(&qb) ; - buffer.open(QIODevice::WriteOnly) ; - image.save(&buffer,"JPEG") ; - - RsVOIPDataChunk voip_chunk ; - voip_chunk.data = malloc(qb.size()); - memcpy(voip_chunk.data,qb.data(),qb.size()) ; - voip_chunk.size = qb.size() ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - _out_queue.push_back(voip_chunk) ; + _frame_size = s ; +} + +JPEGVideoEncoder::JPEGVideoEncoder() + : _ref_frame_max_distance(10),_ref_frame_count(10) +{ +} + +void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size) +{ + // check if we make a diff image, or if we use the full frame. + + QImage encoded_frame ; + + if(_ref_frame_count++ < _ref_frame_max_distance && image.size() == _reference_frame.size()) + { + // compute difference with reference frame. + encoded_frame = image ; + + for(uint32_t i=0;i _out_queue ; + + QSize _frame_size ; }; // Now derive various image encoding/decoding algorithms. @@ -75,16 +77,45 @@ class VideoEncoder class JPEGVideoDecoder: public VideoDecoder { - protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; +protected: + virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + +private: + QImage _last_reference_frame ; }; class JPEGVideoEncoder: public VideoEncoder { - public: - JPEGVideoEncoder() {} +public: + JPEGVideoEncoder() ; - protected: - virtual void encodeData(const QImage& Image) ; +protected: + virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + +private: + QImage _reference_frame ; + uint32_t _ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _ref_frame_count ; }; +class DifferentialWaveletEncoder: public VideoEncoder +{ +public: + DifferentialWaveletEncoder() {} + +protected: + virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + +}; + +class DifferentialWaveletDecoder: public VideoDecoder +{ +public: + DifferentialWaveletDecoder() {} + +protected: + virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + +private: + QImage _last_reference_frame ; +}; diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 7269610b4..25fd0cfbd 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -51,7 +51,9 @@ class RsVOIPPongResult struct RsVOIPDataChunk { - typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVOIPDataType ; + typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00, + RS_VOIP_DATA_TYPE_AUDIO = 0x01, + RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ; void *data ; // create/delete using malloc/free. uint32_t size ; From 3bda2c2660131e5694d8375527c2262c52db53eb Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 10 Aug 2015 22:13:50 -0400 Subject: [PATCH 04/23] added differential frames encoding/decoding, and bandwidth items --- plugins/VOIP/gui/AudioInputConfig.cpp | 57 +++++++++--------- plugins/VOIP/gui/VideoProcessor.cpp | 83 +++++++++++++++++++++----- plugins/VOIP/gui/VideoProcessor.h | 14 +++-- plugins/VOIP/services/rsVOIPItems.cc | 85 +++++++++++++++++++++++++++ plugins/VOIP/services/rsVOIPItems.h | 29 +++++++-- 5 files changed, 213 insertions(+), 55 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index d9931a2af..29a27026a 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -29,17 +29,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -//#include "AudioInput.h" -//#include "AudioOutput.h" #include "AudioStats.h" #include "AudioInputConfig.h" -//#include "Global.h" -//#include "NetworkConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" #include "gui/common/RSGraphWidget.h" +#include "util/RsProtectedTimer.h" + #include #define iroundf(x) ( static_cast(x) ) @@ -104,33 +101,33 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) { std::cerr << "Creating audioInputConfig object" << std::endl; - /* Invoke the Qt Designer generated object setup routine */ - ui.setupUi(this); + /* Invoke the Qt Designer generated object setup routine */ + ui.setupUi(this); - loaded = false; + loaded = false; - inputAudioProcessor = NULL; - inputAudioDevice = NULL; - abSpeech = NULL; - qtTick = NULL; + inputAudioProcessor = NULL; + inputAudioDevice = NULL; + abSpeech = NULL; + qtTick = NULL; - // Create the video pipeline. - // - videoInput = new QVideoInputDevice(this) ; - videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; - - videoDecoder = new JPEGVideoDecoder; - videoDecoder->setDisplayTarget(NULL) ; - - graph_source = new voipGraphSource ; - ui.voipBwGraph->setSource(graph_source); - - graph_source->setVideoInput(videoInput) ; - graph_source->setCollectionTimeLimit(1000*300) ; - graph_source->start() ; - - QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + // Create the video pipeline. + // + videoInput = new QVideoInputDevice(this) ; + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + + videoDecoder = new JPEGVideoDecoder; + videoDecoder->setDisplayTarget(NULL) ; + + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); + + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; } void AudioInputConfig::togglePreview(bool b) @@ -178,7 +175,7 @@ void AudioInputConfig::load() //connect( ui.allowIpDeterminationCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleIpDetermination(bool) ) ); //connect( ui.allowTunnelConnectionCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleTunnelConnection(bool) ) ); - qtTick = new QTimer(this); + qtTick = new RsProtectedTimer(this); connect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); qtTick->start(20); /*if (AudioInputRegistrar::qmNew) { diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index c9add1b26..b19ee03d0 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -13,14 +13,15 @@ VideoDecoder::VideoDecoder() } VideoEncoder::VideoEncoder() - :_frame_size(128,128) + :_frame_size(256,256) { } bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) { - std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; +// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; + //encodeData(img,size_hint,encoded_size) ; return true ; } @@ -44,15 +45,51 @@ void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) { - QByteArray qb((char*)encoded_image_data,size) ; - QImage image ; - if(image.loadFromData(qb,"JPEG")) - return image ; - else - { - std::cerr << "image.loadFromData(): returned an error.: " << std::endl; - return QImage() ; - } + static const int HEADER_SIZE = 4 ; + + // read frame type. Use first 4 bytes to give info about content. + + if(size < HEADER_SIZE) + { + std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << size << std::endl; + return QImage() ; + } + + uint32_t flags = encoded_image_data[0] + (encoded_image_data[1] << 8) ; + + // un-compress image data + + QByteArray qb((char*)&encoded_image_data[HEADER_SIZE],(int)size - HEADER_SIZE) ; + QImage image ; + if(!image.loadFromData(qb,"JPEG")) + { + std::cerr << "image.loadFromData(): returned an error.: " << std::endl; + return QImage() ; + } + + // now see if the frame is a differential frame, or just a reference frame. + + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) + { + if(_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return image ; + } + + QImage res = _reference_frame ; + + for(uint32_t i=0;i> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = 0 ; + ((unsigned char *)voip_chunk.data)[3] = 0 ; + + memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; _out_queue.push_back(voip_chunk) ; diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index d78b1ad9d..42124b4f4 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -80,8 +80,11 @@ class JPEGVideoDecoder: public VideoDecoder protected: virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + static const uint32_t HEADER_SIZE = 0x04 ; + + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: - QImage _last_reference_frame ; + QImage _reference_frame ; }; class JPEGVideoEncoder: public VideoEncoder @@ -92,10 +95,13 @@ public: protected: virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + static const uint32_t HEADER_SIZE = 0x04 ; + + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: - QImage _reference_frame ; - uint32_t _ref_frame_max_distance ; // max distance between two reference frames. - uint32_t _ref_frame_count ; + QImage _reference_frame ; + uint32_t _ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _ref_frame_count ; }; class DifferentialWaveletEncoder: public VideoEncoder diff --git a/plugins/VOIP/services/rsVOIPItems.cc b/plugins/VOIP/services/rsVOIPItems.cc index eb8842468..a75fa4c16 100644 --- a/plugins/VOIP/services/rsVOIPItems.cc +++ b/plugins/VOIP/services/rsVOIPItems.cc @@ -76,6 +76,19 @@ std::ostream& RsVOIPProtocolItem::print(std::ostream &out, uint16_t indent) printRsItemEnd(out, "RsVOIPProtocolItem", indent); return out; } +std::ostream& RsVOIPBandwidthItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsVOIPBandwidthItem", indent); + uint16_t int_Indent = indent + 2; + printIndent(out, int_Indent); + out << "flags: " << std::hex << flags << std::dec << std::endl; + + printIndent(out, int_Indent); + out << "speed: " << bytes_per_sec << std::endl; + + printRsItemEnd(out, "RsVOIPBandwidthItem", indent); + return out; +} std::ostream& RsVOIPDataItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsVOIPDataItem", indent); @@ -100,6 +113,14 @@ uint32_t RsVOIPDataItem::serial_size() const return s; } +uint32_t RsVOIPBandwidthItem::serial_size() const +{ + uint32_t s = 8; /* header */ + s += 4; /* flags */ + s += 4; /* bandwidth */ + + return s; +} uint32_t RsVOIPProtocolItem::serial_size() const { uint32_t s = 8; /* header */ @@ -150,6 +171,40 @@ bool RsVOIPProtocolItem::serialise(void *data, uint32_t& pktsize) return ok; } +bool RsVOIPBandwidthItem::serialise(void *data, uint32_t& pktsize) +{ + uint32_t tlvsize = serial_size() ; + uint32_t offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Header: " << ok << std::endl; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= setRawUInt32(data, tlvsize, &offset, flags); + ok &= setRawUInt32(data, tlvsize, &offset, bytes_per_sec); + + if (offset != tlvsize) + { + ok = false; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size Error! " << std::endl; + } + + return ok; +} /* serialise the data to the buffer */ bool RsVOIPDataItem::serialise(void *data, uint32_t& pktsize) { @@ -254,6 +309,36 @@ RsVOIPProtocolItem::RsVOIPProtocolItem(void *data, uint32_t pktsize) if (!ok) throw std::runtime_error("Deserialisation error!") ; } +RsVOIPBandwidthItem::RsVOIPBandwidthItem(void *data, uint32_t pktsize) + : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_VOIP_PLUGIN != getRsItemService(rstype)) || (RS_PKT_SUBTYPE_VOIP_BANDWIDTH != getRsItemSubType(rstype))) + throw std::runtime_error("Wrong packet type!") ; + + if (pktsize < rssize) /* check size */ + throw std::runtime_error("Not enough size!") ; + + bool ok = true; + + /* skip the header */ + offset += 8; + + /* get mandatory parts first */ + ok &= getRawUInt32(data, rssize, &offset, &flags); + ok &= getRawUInt32(data, rssize, &offset, &bytes_per_sec); + + if (offset != rssize) + throw std::runtime_error("Deserialisation error!") ; + + if (!ok) + throw std::runtime_error("Deserialisation error!") ; +} RsVOIPPingItem::RsVOIPPingItem(void *data, uint32_t pktsize) : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_PING) { diff --git a/plugins/VOIP/services/rsVOIPItems.h b/plugins/VOIP/services/rsVOIPItems.h index fa440ef19..7658ea4cc 100644 --- a/plugins/VOIP/services/rsVOIPItems.h +++ b/plugins/VOIP/services/rsVOIPItems.h @@ -55,11 +55,12 @@ const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021; -const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; -const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; -const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; - // 0x04 is unused because of a change in the protocol -const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x05 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; +const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; +const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; + // 0x04,0x05 is unused because of a change in the protocol +const uint8_t RS_PKT_SUBTYPE_VOIP_BANDWIDTH = 0x06 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x07 ; const uint8_t QOS_PRIORITY_RS_VOIP = 9 ; @@ -117,9 +118,27 @@ class RsVOIPDataItem: public RsVOIPItem uint32_t flags ; uint32_t data_size ; + void *voip_data ; }; +class RsVOIPBandwidthItem: public RsVOIPItem +{ + public: + RsVOIPBandwidthItem() :RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) {} + RsVOIPBandwidthItem(void *data,uint32_t size) ; // de-serialization + + virtual bool serialise(void *data,uint32_t& size) ; + virtual uint32_t serial_size() const ; + + virtual ~RsVOIPBandwidthItem() {} + virtual std::ostream& print(std::ostream &out, uint16_t indent = 0); + + uint32_t flags ; // is that incoming or expected bandwidth? + uint32_t bytes_per_sec ; // bandwidth in bytes per sec. +}; + + class RsVOIPProtocolItem: public RsVOIPItem { public: From cd97fd9682aebb365bfd808693f8a23a12923739 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 14 Aug 2015 16:44:20 -0400 Subject: [PATCH 05/23] refactored video processing classes, to allow multiple codecs. Not working yet. --- plugins/VOIP/gui/AudioInputConfig.cpp | 50 +++--- plugins/VOIP/gui/AudioInputConfig.h | 2 +- plugins/VOIP/gui/QVideoDevice.cpp | 10 +- plugins/VOIP/gui/QVideoDevice.h | 5 +- plugins/VOIP/gui/VOIPChatWidgetHolder.cpp | 80 +++++---- plugins/VOIP/gui/VOIPChatWidgetHolder.h | 6 +- plugins/VOIP/gui/VideoProcessor.cpp | 200 ++++++++++++++-------- plugins/VOIP/gui/VideoProcessor.h | 168 +++++++++--------- 8 files changed, 284 insertions(+), 237 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 29a27026a..5f6aacfb8 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -34,6 +34,7 @@ #include "AudioInputConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" +#include "gui/VideoProcessor.h" #include "gui/common/RSGraphWidget.h" #include "util/RsProtectedTimer.h" @@ -99,35 +100,36 @@ voipGraph::voipGraph(QWidget *parent) AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) { - std::cerr << "Creating audioInputConfig object" << std::endl; + std::cerr << "Creating audioInputConfig object" << std::endl; - /* Invoke the Qt Designer generated object setup routine */ - ui.setupUi(this); + /* Invoke the Qt Designer generated object setup routine */ + ui.setupUi(this); - loaded = false; + loaded = false; - inputAudioProcessor = NULL; - inputAudioDevice = NULL; - abSpeech = NULL; - qtTick = NULL; + inputAudioProcessor = NULL; + inputAudioDevice = NULL; + abSpeech = NULL; + qtTick = NULL; - // Create the video pipeline. - // - videoInput = new QVideoInputDevice(this) ; - videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + // Create the video pipeline. + // + videoInput = new QVideoInputDevice(this) ; + videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoDecoder = new JPEGVideoDecoder; - videoDecoder->setDisplayTarget(NULL) ; + videoProcessor = new VideoProcessor() ; + videoProcessor->setDisplayTarget(NULL) ; - graph_source = new voipGraphSource ; - ui.voipBwGraph->setSource(graph_source); + videoInput->setVideoProcessor(videoProcessor) ; - graph_source->setVideoInput(videoInput) ; - graph_source->setCollectionTimeLimit(1000*300) ; - graph_source->start() ; + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); - QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; } void AudioInputConfig::togglePreview(bool b) @@ -135,12 +137,12 @@ void AudioInputConfig::togglePreview(bool b) if(b) { videoInput->setEchoVideoTarget(NULL) ; - videoDecoder->setDisplayTarget(ui.videoDisplay) ; + videoProcessor->setDisplayTarget(ui.videoDisplay) ; } else { videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoDecoder->setDisplayTarget(NULL) ; + videoProcessor->setDisplayTarget(NULL) ; } } @@ -367,7 +369,7 @@ void AudioInputConfig::on_Tick_timeout() { while(videoInput->getNextEncodedPacket(chunk)) { - videoDecoder->receiveEncodedData(static_cast(chunk.data),chunk.size) ; + videoProcessor->receiveEncodedData(chunk) ; chunk.clear() ; } } diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index faaadb63e..e838744b4 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -69,7 +69,7 @@ class AudioInputConfig : public ConfigPage //VideoDecoder *videoDecoder ; //VideoEncoder *videoEncoder ; QVideoInputDevice *videoInput ; - VideoDecoder *videoDecoder ; + VideoProcessor *videoProcessor ; bool loaded; voipGraphSource *graph_source ; diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 3a7ee029e..e9ace94f9 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -11,7 +11,7 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) { _timer = NULL ; _capture_device = NULL ; - _video_encoder = NULL ; + _video_processor = NULL ; _echo_output_device = NULL ; _estimated_bw = 0 ; _total_encoded_size = 0 ; @@ -78,11 +78,11 @@ void QVideoInputDevice::grabFrame() QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - if(_video_encoder != NULL) + if(_video_processor != NULL) { uint32_t encoded_size ; - _video_encoder->addImage(image,0,encoded_size) ; + _video_processor->processImage(image,0,encoded_size) ; std::cerr << "Encoded size = " << encoded_size << std::endl; _total_encoded_size += encoded_size ; @@ -107,8 +107,8 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { - if(_video_encoder) - return _video_encoder->nextPacket(chunk) ; + if(_video_processor) + return _video_processor->nextEncodedPacket(chunk) ; else return false ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 513932487..d12fe5f38 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -2,6 +2,7 @@ #include #include "interface/rsVOIP.h" +#include "gui/VideoProcessor.h" class VideoEncoder ; class CvCapture ; @@ -31,7 +32,7 @@ class QVideoInputDevice: public QObject // Captured images are sent to this encoder. Can be NULL. // - void setVideoEncoder(VideoEncoder *venc) { _video_encoder = venc ; } + void setVideoProcessor(VideoProcessor *venc) { _video_processor = venc ; } // All images received will be echoed to this target. We could use signal/slots, but it's // probably faster this way. Can be NULL. @@ -58,7 +59,7 @@ class QVideoInputDevice: public QObject void networkPacketReady() ; private: - VideoEncoder *_video_encoder ; + VideoProcessor *_video_processor ; QTimer *_timer ; CvCapture *_capture_device ; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index 1d94960d7..4d1f8ff49 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -123,8 +123,7 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n inputAudioDevice = NULL ; inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-) - inputVideoProcessor = new JPEGVideoEncoder ; - outputVideoProcessor = new JPEGVideoDecoder ; + videoProcessor = new VideoProcessor ; // Make a widget with two video devices, one for echo, and one for the talking peer. videoWidget = new QWidget(mChatWidget) ; @@ -144,8 +143,8 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n mChatWidget->addChatHorizontalWidget(videoWidget) ; inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ; - inputVideoDevice->setVideoEncoder(inputVideoProcessor) ; - outputVideoProcessor->setDisplayTarget(outputVideoDevice) ; + inputVideoDevice->setVideoProcessor(videoProcessor) ; + videoProcessor->setDisplayTarget(outputVideoDevice) ; } VOIPChatWidgetHolder::~VOIPChatWidgetHolder() @@ -154,8 +153,7 @@ VOIPChatWidgetHolder::~VOIPChatWidgetHolder() inputAudioDevice->stop() ; delete inputVideoDevice ; - delete inputVideoProcessor ; - delete outputVideoProcessor ; + delete videoProcessor ; button_map::iterator it = buttonMapTakeVideo.begin(); while (it != buttonMapTakeVideo.end()) { @@ -287,42 +285,50 @@ void VOIPChatWidgetHolder::toggleVideoCapture() void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array) { - if (!videoCaptureToggleButton->isChecked()) { - if (mChatWidget) { - QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); - if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId - button_map::iterator it = buttonMapTakeVideo.find(buttonName); - if (it == buttonMapTakeVideo.end()){ - mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() - , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); - RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); - button->setToolTip(tr("Activate camera")); - button->setStyleSheet(QString("border: 1px solid #199909;") - .append("font-size: 12pt; color: white;") - .append("min-width: 128px; min-height: 24px;") - .append("border-radius: 6px;") - .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " - "stop: 0 #22c70d, stop: 1 #116a06);") + if (!videoCaptureToggleButton->isChecked()) + { + if (mChatWidget) { + QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); + if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId + button_map::iterator it = buttonMapTakeVideo.find(buttonName); + if (it == buttonMapTakeVideo.end()){ + mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() + , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); + RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); + button->setToolTip(tr("Activate camera")); + button->setStyleSheet(QString("border: 1px solid #199909;") + .append("font-size: 12pt; color: white;") + .append("min-width: 128px; min-height: 24px;") + .append("border-radius: 6px;") + .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " + "stop: 0 #22c70d, stop: 1 #116a06);") - ); + ); - button->updateImage(); + button->updateImage(); - connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); - connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); - connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); + connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); + connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); + connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); - buttonMapTakeVideo.insert(buttonName, button); - } - } + buttonMapTakeVideo.insert(buttonName, button); + } + } - //TODO make a sound for the incoming call -// soundManager->play(VOIP_SOUND_INCOMING_CALL); - if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); + //TODO make a sound for the incoming call + // soundManager->play(VOIP_SOUND_INCOMING_CALL); + if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); - } else { - outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ; - } + } + else + { + RsVOIPDataChunk chunk ; + chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + chunk.size = array->size() ; + chunk.data = array->data() ; + + videoProcessor->receiveEncodedData(chunk) ; + } } void VOIPChatWidgetHolder::botMouseEnter() @@ -359,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave() void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) { - inputVideoProcessor->setMaximumFrameRate(bytes_per_sec) ; + videoProcessor->setMaximumFrameRate(bytes_per_sec) ; } void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.h b/plugins/VOIP/gui/VOIPChatWidgetHolder.h index 791a2dcf8..350f49f2c 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.h +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.h @@ -34,8 +34,7 @@ class QAudioInput; class QAudioOutput; class QVideoInputDevice ; class QVideoOutputDevice ; -class VideoEncoder ; -class VideoDecoder ; +class VideoProcessor ; #define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call" @@ -82,8 +81,7 @@ protected: QWidget *videoWidget ; // pointer to call show/hide - VideoEncoder *inputVideoProcessor; - VideoDecoder *outputVideoProcessor; + VideoProcessor *videoProcessor; // Additional buttons to the chat bar QToolButton *audioListenToggleButton ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index b19ee03d0..b3d0a7c27 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -7,127 +8,182 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" -VideoDecoder::VideoDecoder() +VideoProcessor::VideoProcessor() + :_encoded_frame_size(256,256) { - _output_device = NULL ; + _decoded_output_device = NULL ; } -VideoEncoder::VideoEncoder() - :_frame_size(256,256) +bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) { + VideoCodec *codec ; + + switch(_encoding_current_codec) + { + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; + break ; + default: + codec = NULL ; + } + + // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; + + void *data = NULL; + encoded_size = 0 ; + + if(codec) + { + RsVOIPDataChunk chunk ; + + codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; + + if(chunk.size == 0) // the codec might be buffering the frame for compression reasons + return true ; + + _encoded_out_queue.push_back(chunk) ; + + return true ; + } + else + return false ; } -bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) { -// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; - //encodeData(img,size_hint,encoded_size) ; - - return true ; -} - -bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk) -{ - if(_out_queue.empty()) + if(_encoded_out_queue.empty()) return false ; - chunk = _out_queue.front() ; - _out_queue.pop_front() ; + chunk = _encoded_out_queue.front() ; + _encoded_out_queue.pop_front() ; return true ; } -void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) +void VideoProcessor::setInternalFrameSize(QSize s) { - if(_output_device) - _output_device->showFrame(decodeData(data,size)) ; + _encoded_frame_size = s ; } -QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) +void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) { static const int HEADER_SIZE = 4 ; // read frame type. Use first 4 bytes to give info about content. + // + // Byte Meaning Values + // 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec + // CODEC_ID_DDWT_VIDEO Differential wavelet compression + // + // 01 Unused Might be useful later + // + // 0203 Flags Codec specific flags. + // - if(size < HEADER_SIZE) + if(chunk.size < HEADER_SIZE) { - std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << size << std::endl; - return QImage() ; + std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << chunk.size << std::endl; + return ; } - uint32_t flags = encoded_image_data[0] + (encoded_image_data[1] << 8) ; + uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; - // un-compress image data + VideoCodec *codec ; - QByteArray qb((char*)&encoded_image_data[HEADER_SIZE],(int)size - HEADER_SIZE) ; - QImage image ; - if(!image.loadFromData(qb,"JPEG")) + switch(codid) { - std::cerr << "image.loadFromData(): returned an error.: " << std::endl; - return QImage() ; + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; + break ; + default: + codec = NULL ; } + QImage img ; - // now see if the frame is a differential frame, or just a reference frame. + if(codec != NULL) + codec->decodeData(chunk,img) ; - if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) - { - if(_reference_frame.size() != image.size()) - { - std::cerr << "Bad reference frame!" << std::endl; - return image ; - } - - QImage res = _reference_frame ; - - for(uint32_t i=0;ishowFrame(img) ; } -void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) +void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) { std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; } -void VideoEncoder::setInternalFrameSize(QSize s) -{ - _frame_size = s ; -} +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +////////////////////////////////////////////////////////////////////////////////////////////////////////////// -JPEGVideoEncoder::JPEGVideoEncoder() - : _ref_frame_max_distance(50),_ref_frame_count(50) +JPEGVideo::JPEGVideo() + : _encoded_ref_frame_max_distance(50),_encoded_ref_frame_count(50) { } -void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size) +bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + // now see if the frame is a differential frame, or just a reference frame. + + uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + + assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ; + + // un-compress image data + + QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ; + + if(!image.loadFromData(qb,"JPEG")) + { + std::cerr << "image.loadFromData(): returned an error.: " << std::endl; + return false ; + } + + + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) + { + if(_decoded_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return false ; + } + + QImage res = _decoded_reference_frame ; + + for(uint32_t i=0;i> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = 0 ; - ((unsigned char *)voip_chunk.data)[3] = 0 ; + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - _out_queue.push_back(voip_chunk) ; - - encoded_size = voip_chunk.size ; + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 42124b4f4..d103207af 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -6,58 +6,91 @@ class QVideoOutputDevice ; +class VideoCodec +{ +public: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0; +}; + +// Now derive various image encoding/decoding algorithms. +// + +class JPEGVideo: public VideoCodec +{ +public: + JPEGVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + + static const uint32_t HEADER_SIZE = 0x04 ; + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; +private: + QImage _decoded_reference_frame ; + QImage _encoded_reference_frame ; + + uint32_t _encoded_ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _encoded_ref_frame_count ; +}; + +class DifferentialWaveletVideo: public VideoCodec +{ +public: + DifferentialWaveletVideo() {} + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) { return true ; } + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) { return true ; } + +private: + QImage _last_reference_frame ; +}; + // This class decodes video from a stream. It keeps a queue of // decoded frame that needs to be retrieved using the getNextImage() method. // -class VideoDecoder +class VideoProcessor { public: - VideoDecoder() ; - virtual ~VideoDecoder() {} + VideoProcessor() ; + virtual ~VideoProcessor() {} + enum CodecId { + VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, + VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001, + VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002 + }; + +// ===================================================================================== +// =------------------------------------ DECODING -------------------------------------= +// ===================================================================================== + // Gets the next image to be displayed. Once returned, the image should // be cleared from the incoming queue. // - void setDisplayTarget(QVideoOutputDevice *odev) { _output_device = odev ; } - - virtual void receiveEncodedData(const unsigned char *data,uint32_t size) ; + void setDisplayTarget(QVideoOutputDevice *odev) { _decoded_output_device = odev ; } + virtual void receiveEncodedData(const RsVOIPDataChunk& chunk) ; // returns the current (measured) frame rate in bytes per second. // - uint32_t currentFrameRate() const; + uint32_t currentDecodingFrameRate() const; private: - QVideoOutputDevice *_output_device ; + QVideoOutputDevice *_decoded_output_device ; + std::list _decoded_image_queue ; - std::list _image_queue ; - - // Incoming data is processed by a video codec and converted into images. - // - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) = 0 ; - -// // This buffer accumulated incoming encoded data, until a full packet is obtained, -// // since the stream might not send images at once. When incoming images are decoded, the -// // data is removed from the buffer. -// // -// unsigned char *buffer ; -// uint32_t buffer_size ; -}; - -// This class encodes video using a video codec (possibly homemade, or based on existing codecs) -// and produces a data stream that is sent to the network transfer service (e.g. p3VOIP). -// -class VideoEncoder -{ +// ===================================================================================== +// =------------------------------------ ENCODING -------------------------------------= +// ===================================================================================== + public: - VideoEncoder() ; - virtual ~VideoEncoder() {} - // Takes the next image to be encoded. // - bool addImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - - bool packetReady() const { return !_out_queue.empty() ; } - bool nextPacket(RsVOIPDataChunk& ) ; + bool processImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } + bool nextEncodedPacket(RsVOIPDataChunk& ) ; // Used to tweak the compression ratio so that the video can stream ok. // @@ -65,63 +98,16 @@ class VideoEncoder void setInternalFrameSize(QSize) ; protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) =0; - - std::list _out_queue ; + std::list _encoded_out_queue ; + QSize _encoded_frame_size ; + +// ===================================================================================== +// =------------------------------------- Codecs --------------------------------------= +// ===================================================================================== - QSize _frame_size ; + JPEGVideo _jpeg_video_codec ; + DifferentialWaveletVideo _ddwt_video_codec ; + + uint16_t _encoding_current_codec ; }; -// Now derive various image encoding/decoding algorithms. -// - -class JPEGVideoDecoder: public VideoDecoder -{ -protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; - - static const uint32_t HEADER_SIZE = 0x04 ; - - static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; -private: - QImage _reference_frame ; -}; - -class JPEGVideoEncoder: public VideoEncoder -{ -public: - JPEGVideoEncoder() ; - -protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - - static const uint32_t HEADER_SIZE = 0x04 ; - - static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; -private: - QImage _reference_frame ; - uint32_t _ref_frame_max_distance ; // max distance between two reference frames. - uint32_t _ref_frame_count ; -}; - -class DifferentialWaveletEncoder: public VideoEncoder -{ -public: - DifferentialWaveletEncoder() {} - -protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - -}; - -class DifferentialWaveletDecoder: public VideoDecoder -{ -public: - DifferentialWaveletDecoder() {} - -protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; - -private: - QImage _last_reference_frame ; -}; From 5aac92fc97bf976a954d5381f4bce168b2015ff7 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 14 Aug 2015 20:15:44 -0400 Subject: [PATCH 06/23] fixed bug in differential frame encoding --- plugins/VOIP/gui/VideoProcessor.cpp | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index b3d0a7c27..7a43058df 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,6 +12,7 @@ VideoProcessor::VideoProcessor() :_encoded_frame_size(256,256) { _decoded_output_device = NULL ; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -47,7 +48,10 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& return true ; } else + { + std::cerr << "No codec for codec ID = " << _encoding_current_codec << ". Please call VideoProcessor::setCurrentCodec()" << std::endl; return false ; + } } bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) @@ -120,7 +124,7 @@ void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) ////////////////////////////////////////////////////////////////////////////////////////////////////////////// JPEGVideo::JPEGVideo() - : _encoded_ref_frame_max_distance(50),_encoded_ref_frame_count(50) + : _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10) { } @@ -146,19 +150,25 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) { - if(_decoded_reference_frame.size() != image.size()) - { - std::cerr << "Bad reference frame!" << std::endl; - return false ; - } - + if(_decoded_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return false ; + } + QImage res = _decoded_reference_frame ; for(uint32_t i=0;i Date: Fri, 14 Aug 2015 22:44:39 -0400 Subject: [PATCH 07/23] restored original params for JPEG codec --- plugins/VOIP/gui/VideoProcessor.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7a43058df..4e1979a5d 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -9,7 +9,7 @@ #include "QVideoDevice.h" VideoProcessor::VideoProcessor() - :_encoded_frame_size(256,256) + :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; @@ -40,6 +40,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; + encoded_size = chunk.size ; + if(chunk.size == 0) // the codec might be buffering the frame for compression reasons return true ; @@ -146,7 +148,6 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "image.loadFromData(): returned an error.: " << std::endl; return false ; } - if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) { @@ -187,6 +188,9 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa for(uint32_t i=0;i> 8) & 0xff ; ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - + memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; - + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; From d28c1898fd74d09c455a34f1754bc0fb9916e207 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 16 Aug 2015 22:59:49 -0400 Subject: [PATCH 08/23] first workign implementation of wavelet-based codec --- plugins/VOIP/gui/DaubechyWavelets.h | 267 ++++++++++++++++++++++++++++ plugins/VOIP/gui/VideoProcessor.cpp | 240 ++++++++++++++++++++++++- plugins/VOIP/gui/VideoProcessor.h | 25 ++- 3 files changed, 519 insertions(+), 13 deletions(-) create mode 100644 plugins/VOIP/gui/DaubechyWavelets.h diff --git a/plugins/VOIP/gui/DaubechyWavelets.h b/plugins/VOIP/gui/DaubechyWavelets.h new file mode 100644 index 000000000..c69f025b0 --- /dev/null +++ b/plugins/VOIP/gui/DaubechyWavelets.h @@ -0,0 +1,267 @@ +#pragma once + +#include +#include +#include +#include +#include + +#ifdef USE_SSE_INSTRUCTIONS +#include +#endif + +template class DaubechyWavelets +{ + public: + typedef enum { DWT_DAUB02=2, DWT_DAUB04=4, DWT_DAUB12=12, DWT_DAUB20=20 } WaveletType ; + typedef enum { DWT_FORWARD=1, DWT_BACKWARD=0 } TransformType ; + + static void DWT2D(FLOAT *data,unsigned long int W,unsigned long int H,WaveletType type,TransformType tr) + { + unsigned long int nn[2] = {W,H} ; + wtn(&data[-1], &nn[-1],2, tr, waveletFilter(type), pwt) ; + } + static void DWT1D(FLOAT *data,unsigned long int W,WaveletType type,TransformType tr) + { + unsigned long int nn[1] = {W} ; + wtn(&data[-1], &nn[-1],1, tr, waveletFilter(type), pwt) ; + } + + + private: + class wavefilt + { + public: + wavefilt(int n) + { + int k; + FLOAT sig = -1.0; + static const FLOAT c2[5]={ 0.0, sqrt(2.0)/2.0, sqrt(2.0)/2.0, 0.0, 0.0 }; + + static const FLOAT c4[5]={ 0.0, 0.4829629131445341, 0.8365163037378079, 0.2241438680420134,-0.1294095225512604 }; + + static const FLOAT c12[13]={0.0,0.111540743350, 0.494623890398, 0.751133908021, + 0.315250351709,-0.226264693965,-0.129766867567, + 0.097501605587, 0.027522865530,-0.031582039318, + 0.000553842201, 0.004777257511,-0.001077301085}; + + static const FLOAT c20[21]={0.0,0.026670057901, 0.188176800078, 0.527201188932, + 0.688459039454, 0.281172343661,-0.249846424327, + -0.195946274377, 0.127369340336, 0.093057364604, + -0.071394147166,-0.029457536822, 0.033212674059, + 0.003606553567,-0.010733175483, 0.001395351747, + 0.001992405295,-0.000685856695,-0.000116466855, + 0.000093588670,-0.000013264203 }; + + ncof= (n==2)?4:n; + const FLOAT *tmpcc ; + cc.resize(ncof+1) ; + cr.resize(ncof+1) ; + + if (n == 2) + { + tmpcc=c2; + cc[1] = tmpcc[1] ; + cc[2] = tmpcc[2] ; + cc[3] = 0.0f ; + cc[4] = 0.0f ; + cr[1] = tmpcc[1] ; + cr[2] =-tmpcc[2] ; + cr[3] = 0.0f ; + cr[4] = 0.0f ; + + ioff = joff = -1 ; + } + else + { + if (n == 4) + tmpcc=c4; + else if (n == 12) + tmpcc=c12; + else if (n == 20) + tmpcc=c20; + else + throw std::runtime_error("unimplemented value n in pwtset"); + + for (k=1;k<=n;k++) + { + cc[k] = tmpcc[k] ; + cr[ncof+1-k]=sig*tmpcc[k]; + sig = -sig; + } + ioff = joff = -(n >> 1); + } + } + + ~wavefilt() {} + + int ncof,ioff,joff; + std::vector cc; + std::vector cr; + } ; + + static const wavefilt& waveletFilter(WaveletType type) + { + static wavefilt *daub02filt = NULL ; + static wavefilt *daub04filt = NULL ; + static wavefilt *daub12filt = NULL ; + static wavefilt *daub20filt = NULL ; + + switch(type) + { + case DWT_DAUB02: if(daub02filt == NULL) + daub02filt = new wavefilt(2) ; + return *daub02filt ; + + case DWT_DAUB04: if(daub04filt == NULL) + daub04filt = new wavefilt(4) ; + return *daub04filt ; + + case DWT_DAUB12: if(daub12filt == NULL) + daub12filt = new wavefilt(12) ; + return *daub12filt ; + + case DWT_DAUB20: if(daub20filt == NULL) + daub20filt = new wavefilt(20) ; + return *daub20filt ; + + default: + throw std::runtime_error("Unknown wavelet type.") ; + } + } + + static void pwt(FLOAT a[], unsigned long n, int isign,const wavefilt& wfilt) + { +/********************** BEGIN SIGNED PART *************************/ +/** md5sum = 2b9e1e38ac690f50806873cdb4a061ea **/ +/** Validation date = 08/10/10 **/ +/******************************************************************/ + unsigned long i,ii,ni,nj ; + + if (n < 4) + return; + + FLOAT *wksp=new FLOAT[n+1];//vector(1,n); + FLOAT ai,ai1 ; + unsigned long int nmod=wfilt.ncof*n; + unsigned long int n1=n-1; + unsigned long int nh=n >> 1; + + memset(wksp,0,(n+1)*sizeof(FLOAT)) ; + + if (isign == DWT_FORWARD) + for (ii=1,i=1;i<=n;i+=2,ii++) + { + ni=i+nmod+wfilt.ioff; + nj=i+nmod+wfilt.joff; + +#ifdef USE_SSE_INSTRUCTIONS +#warning Using SSE2 Instruction set for wavelet internal loops + for (int k=1;k<=wfilt.ncof;k+=4) + { + int jf=ni+k; + int jr=nj+k; + + sse_block w1(wfilt.cc[k],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) ; + sse_block w2(wfilt.cr[k],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) ; + + sse_block a1( a[1+((jf+0)&n1)], a[1+((jf+1)&n1)], a[1+((jf+2)&n1)], a[1+((jf+3)&n1)]) ; + sse_block a2( a[1+((jr+0)&n1)], a[1+((jr+1)&n1)], a[1+((jr+2)&n1)], a[1+((jr+3)&n1)]) ; + + sse_block wk1( w1*a1 ) ; + sse_block wk2( w2*a2 ) ; + + wksp[ii ] += wk1.sum() ; + wksp[ii+nh] += wk2.sum() ; + } +#else + for (int k=1;k<=wfilt.ncof;k++) + { + int jf=n1 & (ni+k); + int jr=n1 & (nj+k); + wksp[ii] += wfilt.cc[k]*a[jf+1]; + wksp[ii+nh] += wfilt.cr[k]*a[jr+1]; + } +#endif + } + else + for (ii=1,i=1;i<=n;i+=2,ii++) + { + ai=a[ii]; + ai1=a[ii+nh]; + ni=i+nmod+wfilt.ioff; + nj=i+nmod+wfilt.joff; + +#ifdef USE_SSE_INSTRUCTIONS + sse_block ai_sse( ai,ai,ai,ai ) ; + sse_block ai1_sse( ai1,ai1,ai1,ai1 ) ; + + for (int k=1;k<=wfilt.ncof;k+=4) + { + int jf=ni+k ; + int jr=nj+k ; // in fact we have jf==jr, so the code is simpler. + + sse_block w1(wksp[1+((jf+0) & n1)],wksp[1+((jf+1) & n1)],wksp[1+((jf+2) & n1)],wksp[1+((jf+3) & n1)]) ; + + w1 += sse_block(wfilt.cc[k+0],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) * ai_sse ; + w1 += sse_block(wfilt.cr[k+0],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) * ai1_sse ; + + wksp[1+((jr+0) & n1)] = w1[0] ; + wksp[1+((jr+1) & n1)] = w1[1] ; + wksp[1+((jr+2) & n1)] = w1[2] ; + wksp[1+((jr+3) & n1)] = w1[3] ; + } +#else + for (int k=1;k<=wfilt.ncof;++k) + { + wksp[(n1 & (ni+k))+1] += wfilt.cc[k]*ai; + wksp[(n1 & (nj+k))+1] += wfilt.cr[k]*ai1; + } +#endif + } + + for (uint j=1;j<=n;j++) + a[j]=wksp[j]; + + delete[] wksp ;//free_vector(wksp,1,n); +/********************** END SIGNED PART *************************/ + } + + static void wtn(FLOAT a[], unsigned long nn[], int ndim, int isign, const wavefilt& w,void (*wtstep)(FLOAT [], unsigned long, int,const wavefilt&)) + { + unsigned long i1,i2,i3,k,n,nnew,nprev=1,nt,ntot=1; + int idim; + FLOAT *wksp; + + for (idim=1;idim<=ndim;idim++) + ntot *= nn[idim]; + + wksp=new FLOAT[ntot+1] ; //vector(1,ntot); + + for (idim=1;idim<=ndim;idim++) + { + n=nn[idim]; + nnew=n*nprev; + + if (n > 4) + for (i2=0;i2=4;nt >>= 1) + (*wtstep)(wksp,nt,isign,w); + else + for(nt=4;nt<=n;nt <<= 1) + (*wtstep)(wksp,nt,isign,w); + + for (i3=i1+i2,k=1;k<=n;k++,i3+=nprev) a[i3]=wksp[k]; + } + + nprev=nnew; + } + delete[] wksp ;//free_vector(wksp,1,ntot); + } +}; + diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 4e1979a5d..9f4ac7a7c 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -7,12 +7,14 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" +#include "DaubechyWavelets.h" VideoProcessor::VideoProcessor() :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; - _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -31,7 +33,6 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - void *data = NULL; encoded_size = 0 ; if(codec) @@ -111,6 +112,8 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) if(codec != NULL) codec->decodeData(chunk,img) ; + else + std::cerr << "Unknown decoding codec: " << codid << std::endl; if(_decoded_output_device) _decoded_output_device->showFrame(img) ; @@ -159,7 +162,7 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) QImage res = _decoded_reference_frame ; - for(uint32_t i=0;i> 8) & 0xff ; - memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],qb.data(),qb.size()) ; voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; @@ -230,3 +233,230 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa return true ; } + +bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +{ + static const int WAVELET_IMG_SIZE = 128 ; + static const float W_THRESHOLD = 0.005 ; // low quality + //static const float W_THRESHOLD = 0.0001; // high quality + //static const float W_THRESHOLD = 0.0005; // medium quality + + static const int W2 = WAVELET_IMG_SIZE ; + static const int H2 = WAVELET_IMG_SIZE ; + + assert(image.width() == W2) ; + assert(image.height() == H2) ; + + float *temp = new float[W2*H2] ; + + std::cerr << " codec type: wavelets." << std::endl; + + // We should perform some interpolation here ;-) + // + for(int i=0;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_FORWARD) ; + + // Now estimate the max energy in the W coefs, and only keep the largest. + + float mx = 0.0f ; + for(int i=0;i compressed_values ; + compressed_values.reserve(W2*H2) ; + + for(int i=0;i= W_THRESHOLD*mx) // This needs to be improved. Wavelets do not all have the same visual impact. + { + // add one value, using 16 bits for coordinates and 16 bits for the value. + + compressed_values.push_back((uint16_t)i) ; + compressed_values.push_back(quantize_16b(temp[i],mx)) ; + + //float f2 = from_quantized_16b(quantize_16b(temp[i],mx),mx) ; + + //if(fabs(f2 - temp[i]) >= 0.01*(fabs(temp[i])+fabs(f2))) + //std::cerr << " before: " << temp[i] << ", quantised=" << quantize_16b(temp[i],mx)<< ", after: " << f2 << std::endl; + } + delete[] temp ; + + // Serialise all values into a memory buffer. This needs to be taken care of because of endian issues. + + int compressed_size = 4 + compressed_values.size()*2 ; + + std::cerr << " threshold : " << W_THRESHOLD << std::endl; + std::cerr << " values kept: " << compressed_values.size()/2 << std::endl; + std::cerr << " compression: " << compressed_size/float(W2*H2*3)*100 << " %" << std::endl; + + voip_chunk.data = malloc(HEADER_SIZE + compressed_size) ; + + // build header + uint32_t flags = 0 ; + + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + unsigned char *compressed_mem = &((unsigned char *)voip_chunk.data)[HEADER_SIZE] ; + serialise_ufloat(compressed_mem,mx) ; + + for(uint32_t i=0;i> 8 ; + } + + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + voip_chunk.size = HEADER_SIZE + compressed_size ; + + return true ; +} + +bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + static const int WAVELET_IMG_SIZE = 128 ; + + static const int W2 = WAVELET_IMG_SIZE ; + static const int H2 = WAVELET_IMG_SIZE ; + + float *temp = new float[W2*H2] ; + + const unsigned char *compressed_mem = &static_cast(chunk.data)[HEADER_SIZE] ; + int compressed_size = chunk.size - HEADER_SIZE; + + memset(temp,0,W2*H2*sizeof(float)) ; + float M = deserialise_ufloat(compressed_mem); + +#ifdef VOIP_CODEC_DEBUG + std::cerr << " codec type: wavelets." << std::endl; + std::cerr << " max coef: " << M << std::endl; +#endif + + for(int i=4;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_BACKWARD) ; + +#ifdef VOIP_CODEC_DEBUG + std::cerr << " resizing image to: " << w << "x" << h << std::endl; +#endif + + image = QImage(W2,H2,QImage::Format_RGB32) ; + + int indx = 0 ; + + for(int j=0;j1023), and p is coded on 6 bits (0->63). + // Packing [mp] into a 16bit uint16_t. M is the maximum coefficient over the quantization + // process. + // + // So this represents numbers from M * 1 * 2^{-73} to M + // + // All calculatoins are performed on x/M*2^10 + // + static const float LOG2 = log(2.0f) ; + + int m,p ; + + if(fabs(x) < 1e-8*M) + { + m = 0 ; + p = 0 ; + } + else + { + float log2f = log(fabsf(x)/M)/LOG2 ; + int mexp = (int)floor(MANTISSE_BITS - log2f) ; + + m = (int)floor(pow(2.0f,mexp+log2f)) ; + p = mexp ; + + if(p > (1<> EXPONENT_BITS ; + + if(p > 10) + return M * m / 1024.0f / (float)(1 << (p-10)) ; + else + return M * m / (float)(1 << p) ; +} + +void WaveletVideo::serialise_ufloat(unsigned char *mem, float f) +{ + if(f < 0.0f) + { + std::cerr << "(EE) Cannot serialise invalid negative float value " << f << " in " << __PRETTY_FUNCTION__ << std::endl; + return ; + } + // This serialisation is quite accurate. The max relative error is approx. + // 0.01% and most of the time less than 1e-05% The error is well distributed + // over numbers also. + // + uint32_t n = (f < 1e-7)?(~(uint32_t)0): ((uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0)))) ; + + mem[0] = n & 0xff ; n >>= 8 ; + mem[1] = n & 0xff ; n >>= 8 ; + mem[2] = n & 0xff ; n >>= 8 ; + mem[3] = n & 0xff ; +} +float WaveletVideo::deserialise_ufloat(const unsigned char *mem) +{ + uint32_t n = mem[3] ; + n = (n << 8) + mem[2] ; + n = (n << 8) + mem[1] ; + n = (n << 8) + mem[0] ; + + return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; +} diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index d103207af..da12d76ae 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -11,6 +11,9 @@ class VideoCodec public: virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0; + +protected: + static const uint32_t HEADER_SIZE = 0x04 ; }; // Now derive various image encoding/decoding algorithms. @@ -25,7 +28,6 @@ protected: virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; - static const uint32_t HEADER_SIZE = 0x04 ; static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: QImage _decoded_reference_frame ; @@ -35,17 +37,24 @@ private: uint32_t _encoded_ref_frame_count ; }; -class DifferentialWaveletVideo: public VideoCodec +class WaveletVideo: public VideoCodec { public: - DifferentialWaveletVideo() {} + WaveletVideo() {} protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) { return true ; } - virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) { return true ; } - + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: - QImage _last_reference_frame ; + + static const int MANTISSE_BITS = 9 ; + static const int EXPONENT_BITS = 6 ; + + static void serialise_ufloat(unsigned char *mem, float f); + static float deserialise_ufloat(const unsigned char *mem); + + static float from_quantized_16b(uint16_t n, float M); + static uint16_t quantize_16b(float x, float M); }; // This class decodes video from a stream. It keeps a queue of @@ -106,7 +115,7 @@ class VideoProcessor // ===================================================================================== JPEGVideo _jpeg_video_codec ; - DifferentialWaveletVideo _ddwt_video_codec ; + WaveletVideo _ddwt_video_codec ; uint16_t _encoding_current_codec ; }; From 043fe537896458a5d2388eb9486796dcd76b88aa Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 19 Aug 2015 21:50:51 -0400 Subject: [PATCH 09/23] added avcodec encoding --- plugins/VOIP/VOIP.pro | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 190 +++++++++++++++++++++++++++- plugins/VOIP/gui/VideoProcessor.h | 27 +++- 3 files changed, 211 insertions(+), 8 deletions(-) diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index e7f054553..86a716309 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -96,4 +96,4 @@ TRANSLATIONS += \ lang/VOIP_tr.ts \ lang/VOIP_zh_CN.ts -LIBS += -lspeex -lspeexdsp +LIBS += -lspeex -lspeexdsp -lavformat -lavcodec -lavutil diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 9f4ac7a7c..66a810b0a 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -9,12 +9,26 @@ #include "QVideoDevice.h" #include "DaubechyWavelets.h" +#include + +extern "C" { +#include + +#include +#include +#include +#include +#include +#include +} + VideoProcessor::VideoProcessor() :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; - //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; - _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -27,6 +41,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& break ; case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; default: codec = NULL ; } @@ -95,7 +111,7 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) } uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; - uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + //uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; VideoCodec *codec ; @@ -105,6 +121,8 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) break ; case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; default: codec = NULL ; } @@ -258,7 +276,7 @@ bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataC std::cerr << " resized image to B&W " << W2 << "x" << H2 << std::endl; - DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_FORWARD) ; + DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_FORWARD) ; // Now estimate the max energy in the W coefs, and only keep the largest. @@ -358,7 +376,7 @@ bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << " values read: " << compressed_size/4-1 << std::endl; #endif - DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_BACKWARD) ; + DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_BACKWARD) ; #ifdef VOIP_CODEC_DEBUG std::cerr << " resizing image to: " << w << "x" << h << std::endl; @@ -460,3 +478,165 @@ float WaveletVideo::deserialise_ufloat(const unsigned char *mem) return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; } + +FFmpegVideo::FFmpegVideo() +{ + codec = NULL ; + frame_buffer = NULL ; + context = NULL ; + + AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO + + uint8_t endcode[] = { 0, 0, 1, 0xb7 }; + + /* find the mpeg1 video encoder */ + codec = avcodec_find_encoder(codec_id); + + if (!codec) + throw("AV codec not found for codec id ") ; + + context = avcodec_alloc_context3(codec); + + if (!context) + throw std::runtime_error("AV: Could not allocate video codec context"); + + /* put sample parameters */ + context->bit_rate = 400000; + /* resolution must be a multiple of two */ + context->width = 352; + context->height = 288; + /* frames per second */ + context->time_base = (AVRational){1,25}; + /* emit one intra frame every ten frames + * check frame pict_type before passing frame + * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I + * then gop_size is ignored and the output of encoder + * will always be I frame irrespective to gop_size + */ + context->gop_size = 10; + context->max_b_frames = 1; + //context->pix_fmt = AV_PIX_FMT_RGB24; + context->pix_fmt = AV_PIX_FMT_YUV420P; + + if (codec_id == AV_CODEC_ID_H264) + av_opt_set(context->priv_data, "preset", "slow", 0); + + /* open it */ + if (avcodec_open2(context, codec, NULL) < 0) + throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); + + frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + + frame_buffer->format = context->pix_fmt; + frame_buffer->width = context->width; + frame_buffer->height = context->height; + + /* the image can be allocated by any means and av_image_alloc() is + * just the most convenient way if av_malloc() is to be used */ + + int ret = av_image_alloc(frame_buffer->data, frame_buffer->linesize, context->width, context->height, context->pix_fmt, 32); + + if (ret < 0) + throw std::runtime_error("AV: Could not allocate raw picture buffer"); + + frame_count = 0 ; +} + +FFmpegVideo::~FFmpegVideo() +{ + avcodec_close(context); + + av_free(context); + + av_freep(&frame_buffer->data[0]); + + free(frame_buffer); +} + + +bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +{ + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; + + QImage input ; + + if(image.width() != frame_buffer->width || image.height() != frame_buffer->height) + input = image.scaled(QSize(frame_buffer->width,frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + else + input = image ; + + /* prepare a dummy image */ + /* Y */ + for (int y = 0; y < context->height; y++) + for (int x = 0; x < context->width; x++) + { + QRgb pix = image.pixel(QPoint(x,y)) ; + + register int R = (pix >> 16) & 0xff ; + register int G = (pix >> 8) & 0xff ; + register int B = (pix >> 0) & 0xff ; + + register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + } + + + frame_buffer->pts = frame_count++; + + /* encode the image */ + + int got_output = 0; + + AVFrame *frame = frame_buffer ; + +// do +// { + int ret = avcodec_encode_video2(context, &pkt, frame, &got_output) ; + + if (ret < 0) + { + std::cerr << "Error encoding frame!" << std::endl; + return false ; + } +// frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer +// +// } while(got_output) ; + + if(got_output) + { + voip_chunk.data = pkt.data ; + voip_chunk.size = pkt.size ; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } + else + { + voip_chunk.data = NULL; + voip_chunk.size = 0; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "No output produced." << std::endl; + } + + + pkt.data = NULL ; // transfer ownership to chunk + pkt.size = 0 ; + + av_free_packet(&pkt); + + return true ; +} +bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + return true ; +} + diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index da12d76ae..7cc7c1526 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -57,6 +57,27 @@ private: static uint16_t quantize_16b(float x, float M); }; +struct AVCodec ; +struct AVCodecContext ; +struct AVFrame ; + +class FFmpegVideo: public VideoCodec +{ +public: + FFmpegVideo() ; + ~FFmpegVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + +private: + AVCodec *codec; + AVCodecContext *context; + AVFrame *frame_buffer ; + uint64_t frame_count ; +}; + // This class decodes video from a stream. It keeps a queue of // decoded frame that needs to be retrieved using the getNextImage() method. // @@ -69,7 +90,8 @@ class VideoProcessor enum CodecId { VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001, - VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002 + VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002, + VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO = 0x0003 }; // ===================================================================================== @@ -114,8 +136,9 @@ class VideoProcessor // =------------------------------------- Codecs --------------------------------------= // ===================================================================================== - JPEGVideo _jpeg_video_codec ; + JPEGVideo _jpeg_video_codec ; WaveletVideo _ddwt_video_codec ; + FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; }; From 9798c78e2091b1edd6a7be7427aea707dbd67bc2 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 19 Aug 2015 22:17:50 -0400 Subject: [PATCH 10/23] added decoding of avcodec --- plugins/VOIP/gui/VideoProcessor.cpp | 237 ++++++++++++++++------------ plugins/VOIP/gui/VideoProcessor.h | 11 +- 2 files changed, 146 insertions(+), 102 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 66a810b0a..97ef75271 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -481,162 +481,203 @@ float WaveletVideo::deserialise_ufloat(const unsigned char *mem) FFmpegVideo::FFmpegVideo() { - codec = NULL ; - frame_buffer = NULL ; - context = NULL ; + // Encoding + + encoding_codec = NULL ; + encoding_frame_buffer = NULL ; + encoding_context = NULL ; AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO uint8_t endcode[] = { 0, 0, 1, 0xb7 }; /* find the mpeg1 video encoder */ - codec = avcodec_find_encoder(codec_id); + encoding_codec = avcodec_find_encoder(codec_id); - if (!codec) - throw("AV codec not found for codec id ") ; + if (!encoding_codec) throw("AV codec not found for codec id ") ; - context = avcodec_alloc_context3(codec); + encoding_context = avcodec_alloc_context3(encoding_codec); - if (!context) - throw std::runtime_error("AV: Could not allocate video codec context"); + if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); + if (!decoding_context) throw std::runtime_error("AV: Could not allocate video codec decoding context"); /* put sample parameters */ - context->bit_rate = 400000; + encoding_context->bit_rate = 400000; /* resolution must be a multiple of two */ - context->width = 352; - context->height = 288; + encoding_context->width = 352; + encoding_context->height = 288; /* frames per second */ - context->time_base = (AVRational){1,25}; + encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames * check frame pict_type before passing frame * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - context->gop_size = 10; - context->max_b_frames = 1; + encoding_context->gop_size = 10; + encoding_context->max_b_frames = 1; //context->pix_fmt = AV_PIX_FMT_RGB24; - context->pix_fmt = AV_PIX_FMT_YUV420P; + encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; if (codec_id == AV_CODEC_ID_H264) - av_opt_set(context->priv_data, "preset", "slow", 0); + av_opt_set(encoding_context->priv_data, "preset", "slow", 0); /* open it */ - if (avcodec_open2(context, codec, NULL) < 0) + if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); - frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + encoding_frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; - frame_buffer->format = context->pix_fmt; - frame_buffer->width = context->width; - frame_buffer->height = context->height; + encoding_frame_buffer->format = encoding_context->pix_fmt; + encoding_frame_buffer->width = encoding_context->width; + encoding_frame_buffer->height = encoding_context->height; /* the image can be allocated by any means and av_image_alloc() is * just the most convenient way if av_malloc() is to be used */ - int ret = av_image_alloc(frame_buffer->data, frame_buffer->linesize, context->width, context->height, context->pix_fmt, 32); + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); if (ret < 0) throw std::runtime_error("AV: Could not allocate raw picture buffer"); - frame_count = 0 ; + encoding_frame_count = 0 ; + + // Decoding + + + decoding_codec = avcodec_find_decoder(codec_id); + + if (!decoding_codec) + throw("AV codec not found for codec id ") ; + + decoding_context = avcodec_alloc_context3(decoding_codec); + } FFmpegVideo::~FFmpegVideo() { - avcodec_close(context); - - av_free(context); - - av_freep(&frame_buffer->data[0]); - - free(frame_buffer); + avcodec_close(encoding_context); + avcodec_close(decoding_context); + av_free(encoding_context); + av_free(decoding_context); + av_freep(&encoding_frame_buffer->data[0]); + av_freep(&decoding_frame_buffer->data[0]); + free(encoding_frame_buffer); + free(decoding_frame_buffer); } bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - AVPacket pkt ; - av_init_packet(&pkt); - pkt.data = NULL; // packet data will be allocated by the encoder - pkt.size = 0; + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; - QImage input ; + QImage input ; - if(image.width() != frame_buffer->width || image.height() != frame_buffer->height) - input = image.scaled(QSize(frame_buffer->width,frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; - else - input = image ; + if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) + input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + else + input = image ; - /* prepare a dummy image */ - /* Y */ - for (int y = 0; y < context->height; y++) - for (int x = 0; x < context->width; x++) + /* prepare a dummy image */ + /* Y */ + for (int y = 0; y < encoding_context->height; y++) + for (int x = 0; x < encoding_context->width; x++) + { + QRgb pix = image.pixel(QPoint(x,y)) ; + + register int R = (pix >> 16) & 0xff ; + register int G = (pix >> 8) & 0xff ; + register int B = (pix >> 0) & 0xff ; + + register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + } + + + encoding_frame_buffer->pts = encoding_frame_count++; + + /* encode the image */ + + int got_output = 0; + + AVFrame *frame = encoding_frame_buffer ; + + // do + // { + int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; + + if (ret < 0) { - QRgb pix = image.pixel(QPoint(x,y)) ; + std::cerr << "Error encoding frame!" << std::endl; + return false ; + } + // frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer + // + // } while(got_output) ; - register int R = (pix >> 16) & 0xff ; - register int G = (pix >> 8) & 0xff ; - register int B = (pix >> 0) & 0xff ; + if(got_output) + { + voip_chunk.data = pkt.data ; + voip_chunk.size = pkt.size ; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; - register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; - register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } + else + { + voip_chunk.data = NULL; + voip_chunk.size = 0; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "No output produced." << std::endl; } - frame_buffer->pts = frame_count++; + pkt.data = NULL ; // transfer ownership to chunk + pkt.size = 0 ; - /* encode the image */ + av_free_packet(&pkt); - int got_output = 0; - - AVFrame *frame = frame_buffer ; - -// do -// { - int ret = avcodec_encode_video2(context, &pkt, frame, &got_output) ; - - if (ret < 0) - { - std::cerr << "Error encoding frame!" << std::endl; - return false ; - } -// frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer -// -// } while(got_output) ; - - if(got_output) - { - voip_chunk.data = pkt.data ; - voip_chunk.size = pkt.size ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; - } - else - { - voip_chunk.data = NULL; - voip_chunk.size = 0; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "No output produced." << std::endl; - } - - - pkt.data = NULL ; // transfer ownership to chunk - pkt.size = 0 ; - - av_free_packet(&pkt); - - return true ; + return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { + AVPacket pkt ; + + av_init_packet(&pkt); + pkt.data = static_cast(chunk.data); // packet data will be allocated by the encoder + pkt.size = chunk.size; + + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + + if(!got_frame) + return true; + + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int V = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + + register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + register int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128) +))) ; + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 7cc7c1526..c60fc9864 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -72,10 +72,13 @@ protected: virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: - AVCodec *codec; - AVCodecContext *context; - AVFrame *frame_buffer ; - uint64_t frame_count ; + AVCodec *encoding_codec; + AVCodec *decoding_codec; + AVCodecContext *encoding_context; + AVCodecContext *decoding_context; + AVFrame *encoding_frame_buffer ; + AVFrame *decoding_frame_buffer ; + uint64_t encoding_frame_count ; }; // This class decodes video from a stream. It keeps a queue of From 1249423eff6c484abda171822b7c5d46b6e6c84e Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 20 Aug 2015 21:46:28 -0400 Subject: [PATCH 11/23] worked out some bugs. Still not working --- plugins/VOIP/gui/VideoProcessor.cpp | 68 ++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 97ef75271..9f4ebec6e 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -128,10 +129,17 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) } QImage img ; - if(codec != NULL) - codec->decodeData(chunk,img) ; - else + if(codec == NULL) + { std::cerr << "Unknown decoding codec: " << codid << std::endl; + return ; + } + + if(!codec->decodeData(chunk,img)) + { + std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; + return ; + } if(_decoded_output_device) _decoded_output_device->showFrame(img) ; @@ -487,7 +495,8 @@ FFmpegVideo::FFmpegVideo() encoding_frame_buffer = NULL ; encoding_context = NULL ; - AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO + //AVCodecID codec_id = AV_CODEC_ID_H264 ; + AVCodecID codec_id = AV_CODEC_ID_MPEG1VIDEO; uint8_t endcode[] = { 0, 0, 1, 0xb7 }; @@ -526,7 +535,7 @@ FFmpegVideo::FFmpegVideo() if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); - encoding_frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; encoding_frame_buffer->format = encoding_context->pix_fmt; encoding_frame_buffer->width = encoding_context->width; @@ -544,7 +553,7 @@ FFmpegVideo::FFmpegVideo() // Decoding - + decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; decoding_codec = avcodec_find_decoder(codec_id); if (!decoding_codec) @@ -552,6 +561,11 @@ FFmpegVideo::FFmpegVideo() decoding_context = avcodec_alloc_context3(decoding_codec); +// if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) +// decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames + + if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) + throw("AV codec open action failed! ") ; } FFmpegVideo::~FFmpegVideo() @@ -574,6 +588,7 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh pkt.data = NULL; // packet data will be allocated by the encoder pkt.size = 0; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -624,13 +639,22 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh // } while(got_output) ; if(got_output) - { - voip_chunk.data = pkt.data ; - voip_chunk.size = pkt.size ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + { + voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + uint32_t flags = 0; - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; - } + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; + + voip_chunk.size = pkt.size + HEADER_SIZE; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } else { voip_chunk.data = NULL; @@ -640,10 +664,6 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh std::cerr << "No output produced." << std::endl; } - - pkt.data = NULL ; // transfer ownership to chunk - pkt.size = 0 ; - av_free_packet(&pkt); return true ; @@ -653,23 +673,29 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) AVPacket pkt ; av_init_packet(&pkt); - pkt.data = static_cast(chunk.data); // packet data will be allocated by the encoder - pkt.size = chunk.size; + pkt.data = (unsigned char *)memalign(16,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + + pkt.size = chunk.size - HEADER_SIZE; + int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; if(!got_frame) - return true; + return false; image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) { int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int V = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[0] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[0] + x] ; register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; From 4522fe85cc57e72e505eb4dd889b400f7970d5cd Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 20 Aug 2015 22:59:51 -0400 Subject: [PATCH 12/23] fixed encoding --- plugins/VOIP/gui/VideoProcessor.cpp | 102 ++++++++++++++++------------ plugins/VOIP/gui/VideoProcessor.h | 2 + 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 9f4ebec6e..6faab4c80 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -22,9 +22,10 @@ extern "C" { #include #include } +#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(128,128) + :_encoded_frame_size(176,144) { _decoded_output_device = NULL ; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; @@ -135,14 +136,14 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } - if(!codec->decodeData(chunk,img)) - { - std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; - return ; - } - - if(_decoded_output_device) - _decoded_output_device->showFrame(img) ; +// if(!codec->decodeData(chunk,img)) +// { +// std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; +// return ; +// } +// +// if(_decoded_output_device) +// _decoded_output_device->showFrame(img) ; } void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) @@ -513,8 +514,8 @@ FFmpegVideo::FFmpegVideo() /* put sample parameters */ encoding_context->bit_rate = 400000; /* resolution must be a multiple of two */ - encoding_context->width = 352; - encoding_context->height = 288; + encoding_context->width = 176; + encoding_context->height = 144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -560,12 +561,21 @@ FFmpegVideo::FFmpegVideo() throw("AV codec not found for codec id ") ; decoding_context = avcodec_alloc_context3(decoding_codec); + decoding_context->width = encoding_context->width; + decoding_context->height = encoding_context->height; + decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; // if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) // decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) throw("AV codec open action failed! ") ; + + // debug +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; + encoding_debug_file = fopen("tmpvideo.mpg","w") ; +#endif } FFmpegVideo::~FFmpegVideo() @@ -601,19 +611,19 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh for (int y = 0; y < encoding_context->height; y++) for (int x = 0; x < encoding_context->width; x++) { - QRgb pix = image.pixel(QPoint(x,y)) ; + QRgb pix = input.pixel(QPoint(x,y)) ; register int R = (pix >> 16) & 0xff ; register int G = (pix >> 8) & 0xff ; register int B = (pix >> 0) & 0xff ; - register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; - register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; - register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + int Y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,Y)); // Y + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr + encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb } @@ -654,6 +664,10 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; std::cerr << "Output : " << pkt.size << " bytes." << std::endl; +#ifdef DEBUG_MPEG_VIDEO + fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; + fflush(encoding_debug_file) ; +#endif } else { @@ -670,37 +684,37 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - AVPacket pkt ; - - av_init_packet(&pkt); - - pkt.data = (unsigned char *)memalign(16,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; - - pkt.size = chunk.size - HEADER_SIZE; + AVPacket pkt ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; - - if(!got_frame) - return false; - - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - - std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; - - for (int y = 0; y < decoding_frame_buffer->height; y++) + av_init_packet(&pkt); + + pkt.data = (unsigned char *)memalign(32,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + + pkt.size = chunk.size - HEADER_SIZE; + + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + + if(!got_frame) + return false; + + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + + std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + + for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) { int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[0] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; - register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - register int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128) -))) ; image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index c60fc9864..b947c1a0a 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -79,6 +79,8 @@ private: AVFrame *encoding_frame_buffer ; AVFrame *decoding_frame_buffer ; uint64_t encoding_frame_count ; + + FILE *encoding_debug_file ; }; // This class decodes video from a stream. It keeps a queue of From 0bb5a74099d28da1687bf8b688d6e1dc350099dd Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 10:27:50 -0400 Subject: [PATCH 13/23] fixed some bugs. Still crashing in the decoding phase. Don_t know why. --- plugins/VOIP/gui/VideoProcessor.cpp | 110 ++++++++++++++++------------ 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 6faab4c80..ed5436043 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,6 +12,7 @@ #include + extern "C" { #include @@ -136,14 +137,14 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } -// if(!codec->decodeData(chunk,img)) -// { -// std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; -// return ; -// } -// -// if(_decoded_output_device) -// _decoded_output_device->showFrame(img) ; + if(!codec->decodeData(chunk,img)) + { + std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; + return ; + } + + if(_decoded_output_device) + _decoded_output_device->showFrame(img) ; } void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) @@ -509,7 +510,6 @@ FFmpegVideo::FFmpegVideo() encoding_context = avcodec_alloc_context3(encoding_codec); if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); - if (!decoding_context) throw std::runtime_error("AV: Could not allocate video codec decoding context"); /* put sample parameters */ encoding_context->bit_rate = 400000; @@ -554,23 +554,32 @@ FFmpegVideo::FFmpegVideo() // Decoding - decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; decoding_codec = avcodec_find_decoder(codec_id); if (!decoding_codec) throw("AV codec not found for codec id ") ; decoding_context = avcodec_alloc_context3(decoding_codec); + + if(!decoding_context) + throw std::runtime_error("AV: Could not allocate video codec decoding context"); + decoding_context->width = encoding_context->width; decoding_context->height = encoding_context->height; decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; -// if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) -// decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames + if(decoding_codec->capabilities & CODEC_CAP_TRUNCATED) + decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) throw("AV codec open action failed! ") ; + decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); + + //if (ret < 0) + //throw std::runtime_error("AV: Could not allocate raw picture buffer"); + // debug #ifdef DEBUG_MPEG_VIDEO std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; @@ -593,11 +602,6 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - AVPacket pkt ; - av_init_packet(&pkt); - pkt.data = NULL; // packet data will be allocated by the encoder - pkt.size = 0; - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; @@ -635,6 +639,11 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh AVFrame *frame = encoding_frame_buffer ; + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; + // do // { int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; @@ -662,12 +671,13 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.size = pkt.size + HEADER_SIZE; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; fflush(encoding_debug_file) ; #endif + av_free_packet(&pkt); } else { @@ -678,46 +688,56 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh std::cerr << "No output produced." << std::endl; } - av_free_packet(&pkt); - return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - AVPacket pkt ; + std::cerr << "Decoding data of size " << chunk.size << std::endl; - av_init_packet(&pkt); + AVPacket pkt ; + av_init_packet(&pkt); - pkt.data = (unsigned char *)memalign(32,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + pkt.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; - pkt.size = chunk.size - HEADER_SIZE; + pkt.size = chunk.size - HEADER_SIZE; + pkt.pts = AV_NOPTS_VALUE ; + pkt.dts = AV_NOPTS_VALUE ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; - if(!got_frame) - return false; + if(len < 0) + { + std::cerr << "Error decodign frame!" << std::endl; + return false ; + } + if(!got_frame) + return false; - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + assert(pkt.size > 0) ; - std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + av_free_packet(&pkt) ; - for (int y = 0; y < decoding_frame_buffer->height; y++) - for (int x = 0; x < decoding_frame_buffer->width; x++) - { - int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; - - image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; - } + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; - return true ; + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + + return true ; } From 024999ed670391af62f1f6957c0da8eda46bc2bf Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 21:08:27 -0400 Subject: [PATCH 14/23] fixed bug in creating input image when encoding video --- plugins/VOIP/gui/VideoProcessor.cpp | 91 ++++++++++++++++++----------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index ed5436043..aafc092c9 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -58,14 +58,11 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& { RsVOIPDataChunk chunk ; - codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; - - encoded_size = chunk.size ; - - if(chunk.size == 0) // the codec might be buffering the frame for compression reasons - return true ; - - _encoded_out_queue.push_back(chunk) ; + if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) + { + encoded_size = chunk.size ; + _encoded_out_queue.push_back(chunk) ; + } return true ; } @@ -538,6 +535,9 @@ FFmpegVideo::FFmpegVideo() encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + if(!encoding_frame_buffer) + throw std::runtime_error("AV: could not allocate frame buffer.") ; + encoding_frame_buffer->format = encoding_context->pix_fmt; encoding_frame_buffer->width = encoding_context->width; encoding_frame_buffer->height = encoding_context->height; @@ -545,7 +545,8 @@ FFmpegVideo::FFmpegVideo() /* the image can be allocated by any means and av_image_alloc() is * just the most convenient way if av_malloc() is to be used */ - int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, + encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); if (ret < 0) throw std::runtime_error("AV: Could not allocate raw picture buffer"); @@ -575,6 +576,7 @@ FFmpegVideo::FFmpegVideo() throw("AV codec open action failed! ") ; decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); //if (ret < 0) @@ -602,7 +604,7 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -612,20 +614,36 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh /* prepare a dummy image */ /* Y */ - for (int y = 0; y < encoding_context->height; y++) - for (int x = 0; x < encoding_context->width; x++) + for (int y = 0; y < encoding_context->height/2; y++) + for (int x = 0; x < encoding_context->width/2; x++) { - QRgb pix = input.pixel(QPoint(x,y)) ; + QRgb pix00 = input.pixel(QPoint(2*x+0,2*y+0)) ; + QRgb pix01 = input.pixel(QPoint(2*x+0,2*y+1)) ; + QRgb pix10 = input.pixel(QPoint(2*x+1,2*y+0)) ; + QRgb pix11 = input.pixel(QPoint(2*x+1,2*y+1)) ; - register int R = (pix >> 16) & 0xff ; - register int G = (pix >> 8) & 0xff ; - register int B = (pix >> 0) & 0xff ; + int R00 = (pix00 >> 16) & 0xff ; int G00 = (pix00 >> 8) & 0xff ; int B00 = (pix00 >> 0) & 0xff ; + int R01 = (pix01 >> 16) & 0xff ; int G01 = (pix01 >> 8) & 0xff ; int B01 = (pix01 >> 0) & 0xff ; + int R10 = (pix10 >> 16) & 0xff ; int G10 = (pix10 >> 8) & 0xff ; int B10 = (pix10 >> 0) & 0xff ; + int R11 = (pix11 >> 16) & 0xff ; int G11 = (pix11 >> 8) & 0xff ; int B11 = (pix11 >> 0) & 0xff ; - int Y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + int Y00 = (0.257 * R00) + (0.504 * G00) + (0.098 * B00) + 16 ; + int Y01 = (0.257 * R01) + (0.504 * G01) + (0.098 * B01) + 16 ; + int Y10 = (0.257 * R10) + (0.504 * G10) + (0.098 * B10) + 16 ; + int Y11 = (0.257 * R11) + (0.504 * G11) + (0.098 * B11) + 16 ; + + float R = 0.25*(R00+R01+R10+R11) ; + float G = 0.25*(G00+G01+G10+G11) ; + float B = 0.25*(B00+B01+B10+B11) ; + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,Y)); // Y + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y00)); // Y + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y01)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y10)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y11)); // Y + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb } @@ -658,27 +676,29 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh // } while(got_output) ; if(got_output) - { - voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; - uint32_t flags = 0; + { + voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + uint32_t flags = 0; - ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; - ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; - ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; - voip_chunk.size = pkt.size + HEADER_SIZE; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + voip_chunk.size = pkt.size + HEADER_SIZE; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO - fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; - fflush(encoding_debug_file) ; + fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; + fflush(encoding_debug_file) ; #endif - av_free_packet(&pkt); - } + av_free_packet(&pkt); + + return true ; + } else { voip_chunk.data = NULL; @@ -686,9 +706,9 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; std::cerr << "No output produced." << std::endl; + return false ; } - return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { @@ -713,6 +733,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "Error decodign frame!" << std::endl; return false ; } + std::cerr << "Used " << len << " bytes out of " << pkt.size << ". got_frame = " << got_frame << std::endl; if(!got_frame) return false; From f55c115bdbbc7540376e87ee939076eb340ac8b3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 22:39:16 -0400 Subject: [PATCH 15/23] fixed a few bugs. Still not working --- plugins/VOIP/gui/QVideoDevice.cpp | 5 +- plugins/VOIP/gui/VideoProcessor.cpp | 107 ++++++++++++++++++---------- plugins/VOIP/gui/VideoProcessor.h | 6 ++ 3 files changed, 81 insertions(+), 37 deletions(-) diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index e9ace94f9..c3deac879 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -83,8 +83,9 @@ void QVideoInputDevice::grabFrame() uint32_t encoded_size ; _video_processor->processImage(image,0,encoded_size) ; - +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE std::cerr << "Encoded size = " << encoded_size << std::endl; +#endif _total_encoded_size += encoded_size ; time_t now = time(NULL) ; @@ -96,7 +97,9 @@ void QVideoInputDevice::grabFrame() _total_encoded_size = 0 ; _last_bw_estimate_TS = now ; +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE std::cerr << "new bw estimate: " << _estimated_bw << std::endl; +#endif } emit networkPacketReady() ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index aafc092c9..cadef3048 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -577,6 +577,10 @@ FFmpegVideo::FFmpegVideo() decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + av_init_packet(&decoding_buffer); + decoding_buffer.data = NULL ; + decoding_buffer.size = 0 ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); //if (ret < 0) @@ -604,7 +608,7 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -712,53 +716,84 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - std::cerr << "Decoding data of size " << chunk.size << std::endl; + std::cerr << "Decoding data of size " << chunk.size << std::endl; + unsigned char *buff ; - AVPacket pkt ; - av_init_packet(&pkt); + if(decoding_buffer.data != NULL) + { + std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " + << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; - pkt.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; + unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(tmp,0,s+FF_INPUT_BUFFER_PADDING_SIZE) ; - pkt.size = chunk.size - HEADER_SIZE; - pkt.pts = AV_NOPTS_VALUE ; - pkt.dts = AV_NOPTS_VALUE ; + memcpy(tmp,decoding_buffer.data,decoding_buffer.size) ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + free(decoding_buffer.data) ; - if(len < 0) - { - std::cerr << "Error decodign frame!" << std::endl; - return false ; - } - std::cerr << "Used " << len << " bytes out of " << pkt.size << ". got_frame = " << got_frame << std::endl; - if(!got_frame) - return false; + buff = &tmp[decoding_buffer.size] ; + decoding_buffer.size = s ; + decoding_buffer.data = tmp ; + } + else + { + std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; - assert(pkt.size > 0) ; + decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + decoding_buffer.size = chunk.size - HEADER_SIZE ; + memset(decoding_buffer.data,0,decoding_buffer.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - av_free_packet(&pkt) ; + buff = decoding_buffer.data ; + } - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + memcpy(buff,chunk.data,chunk.size - HEADER_SIZE) ; - std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; - for (int y = 0; y < decoding_frame_buffer->height; y++) - for (int x = 0; x < decoding_frame_buffer->width; x++) - { - int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + if(len < 0) + { + std::cerr << "Error decodign frame!" << std::endl; + return false ; + } + std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + if(got_frame) + { + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; - } + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; - return true ; + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; + int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + } + + if(len == decoding_buffer.size) + { + free(decoding_buffer.data) ; + decoding_buffer.data = NULL; + decoding_buffer.size = 0; + } + else if(len != 0) + { + std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; + + memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; + decoding_buffer.size -= len ; + } + + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index b947c1a0a..f5057e5e4 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -4,6 +4,10 @@ #include #include "interface/rsVOIP.h" +extern "C" { +#include +} + class QVideoOutputDevice ; class VideoCodec @@ -60,6 +64,7 @@ private: struct AVCodec ; struct AVCodecContext ; struct AVFrame ; +struct AVPacket ; class FFmpegVideo: public VideoCodec { @@ -78,6 +83,7 @@ private: AVCodecContext *decoding_context; AVFrame *encoding_frame_buffer ; AVFrame *decoding_frame_buffer ; + AVPacket decoding_buffer; uint64_t encoding_frame_count ; FILE *encoding_debug_file ; From 79aac23b6c49107d9a1b73fc066fbed4bf9b9331 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 22:58:36 -0400 Subject: [PATCH 16/23] fixed video codec! --- plugins/VOIP/gui/VideoProcessor.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index cadef3048..51c97b780 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -509,10 +509,10 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 400000; + encoding_context->bit_rate = 200000; /* resolution must be a multiple of two */ - encoding_context->width = 176; - encoding_context->height = 144; + encoding_context->width = 352;//176; + encoding_context->height = 288;//144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -608,7 +608,9 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; +#endif QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -694,8 +696,8 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.size = pkt.size + HEADER_SIZE; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; fflush(encoding_debug_file) ; #endif @@ -716,13 +718,17 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Decoding data of size " << chunk.size << std::endl; +#endif unsigned char *buff ; if(decoding_buffer.data != NULL) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; +#endif uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; @@ -738,7 +744,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) } else { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; +#endif decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; decoding_buffer.size = chunk.size - HEADER_SIZE ; @@ -747,7 +755,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) buff = decoding_buffer.data ; } - memcpy(buff,chunk.data,chunk.size - HEADER_SIZE) ; + memcpy(buff,chunk.data+HEADER_SIZE,chunk.size - HEADER_SIZE) ; int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; @@ -757,13 +765,17 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "Error decodign frame!" << std::endl; return false ; } +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; +#endif if(got_frame) { image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; +#endif for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) @@ -788,7 +800,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) } else if(len != 0) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; +#endif memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; decoding_buffer.size -= len ; From 3f1ebca8032e7efd44389b1f2efb246c99710599 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 24 Aug 2015 21:15:33 -0400 Subject: [PATCH 17/23] added max bandwidth test preview and in/out bw estimate. Still missing the logic inside the codec --- plugins/VOIP/gui/AudioInputConfig.cpp | 9 ++++ plugins/VOIP/gui/AudioInputConfig.h | 5 ++- plugins/VOIP/gui/AudioInputConfig.ui | 35 ++++++++++++++- plugins/VOIP/gui/QVideoDevice.cpp | 32 +++---------- plugins/VOIP/gui/QVideoDevice.h | 6 +-- plugins/VOIP/gui/VOIPChatWidgetHolder.cpp | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 55 +++++++++++++++++++---- plugins/VOIP/gui/VideoProcessor.h | 21 +++++++-- 8 files changed, 120 insertions(+), 45 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 5f6aacfb8..6ef39bce5 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -120,6 +120,8 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) videoProcessor = new VideoProcessor() ; videoProcessor->setDisplayTarget(NULL) ; + videoProcessor->setMaximumBandwidth(ui.availableBW_SB->value()) ; + videoInput->setVideoProcessor(videoProcessor) ; graph_source = new voipGraphSource ; @@ -130,8 +132,15 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) graph_source->start() ; QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + QObject::connect(ui.availableBW_SB,SIGNAL(valueChanged(double)),this,SLOT(updateAvailableBW(double))) ; } +void AudioInputConfig::updateAvailableBW(double r) +{ + std::cerr << "Setting max bandwidth to " << r << " KB/s" << std::endl; + videoProcessor->setMaximumBandwidth((uint32_t)(r*1024)) ; +} + void AudioInputConfig::togglePreview(bool b) { if(b) diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index e838744b4..ef53c28e9 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -93,8 +93,9 @@ class AudioInputConfig : public ConfigPage virtual QPixmap iconPixmap() const { return QPixmap(":/images/talking_on.svg") ; } virtual QString pageName() const { return tr("VOIP") ; } virtual QString helpText() const { return ""; } - - private slots: + +private slots: + void updateAvailableBW(double r); void loadSettings(); void emptyBuffer(); void togglePreview(bool) ; diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 7b77fb3a9..71ed06569 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -7,7 +7,7 @@ 0 0 1155 - 713 + 832 @@ -391,6 +391,39 @@ + + + + + + Available bandwidth: + + + + + + + <html><head/><body><p>Use this field to simulate the maximum bandwidth available so as to preview what the encoded video will look like with the corresponding compression rate.</p></body></html> + + + KB/s + + + 1 + + + 5.000000000000000 + + + 200.000000000000000 + + + 30.000000000000000 + + + + + diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index c3deac879..516933807 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -13,9 +13,6 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _capture_device = NULL ; _video_processor = NULL ; _echo_output_device = NULL ; - _estimated_bw = 0 ; - _total_encoded_size = 0 ; - _last_bw_estimate_TS = time(NULL) ; } void QVideoInputDevice::stop() @@ -80,27 +77,7 @@ void QVideoInputDevice::grabFrame() if(_video_processor != NULL) { - uint32_t encoded_size ; - - _video_processor->processImage(image,0,encoded_size) ; -#ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "Encoded size = " << encoded_size << std::endl; -#endif - _total_encoded_size += encoded_size ; - - time_t now = time(NULL) ; - - if(now > _last_bw_estimate_TS) - { - _estimated_bw = uint32_t(0.75*_estimated_bw + 0.25 * (_total_encoded_size / (float)(now - _last_bw_estimate_TS))) ; - - _total_encoded_size = 0 ; - _last_bw_estimate_TS = now ; - -#ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "new bw estimate: " << _estimated_bw << std::endl; -#endif - } + _video_processor->processImage(image,0) ; emit networkPacketReady() ; } @@ -113,7 +90,12 @@ bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) if(_video_processor) return _video_processor->nextEncodedPacket(chunk) ; else - return false ; + return false ; +} + +uint32_t QVideoInputDevice::currentBandwidth() const +{ + return _video_processor->currentBandwidthOut() ; } QVideoInputDevice::~QVideoInputDevice() diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index d12fe5f38..20a837446 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -45,7 +45,7 @@ class QVideoInputDevice: public QObject // gets the estimated current bandwidth required to transmit the encoded data, in B/s // - uint32_t currentBandwidth() const { return _estimated_bw ; } + uint32_t currentBandwidth() const ; // control @@ -66,9 +66,5 @@ class QVideoInputDevice: public QObject QVideoOutputDevice *_echo_output_device ; std::list _out_queue ; - - uint32_t _estimated_bw ; - time_t _last_bw_estimate_TS; - uint32_t _total_encoded_size ; }; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index 4d1f8ff49..f8760132d 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -365,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave() void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) { - videoProcessor->setMaximumFrameRate(bytes_per_sec) ; + videoProcessor->setMaximumBandwidth(bytes_per_sec) ; } void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 51c97b780..c4934ee0c 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -32,9 +32,19 @@ VideoProcessor::VideoProcessor() //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; + + _estimated_bandwidth_in = 0 ; + _estimated_bandwidth_out = 0 ; + _target_bandwidth_out = 0 ; + + _total_encoded_size_in = 0 ; + _total_encoded_size_out = 0 ; + + _last_bw_estimate_in_TS = time(NULL) ; + _last_bw_estimate_out_TS = time(NULL) ; } -bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { VideoCodec *codec ; @@ -52,17 +62,30 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - encoded_size = 0 ; - if(codec) { RsVOIPDataChunk chunk ; if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) - { - encoded_size = chunk.size ; - _encoded_out_queue.push_back(chunk) ; - } + { + _encoded_out_queue.push_back(chunk) ; + + _total_encoded_size_out += chunk.size ; + } + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_out_TS) + { + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; + + _total_encoded_size_out = 0 ; + _last_bw_estimate_out_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate: " << _estimated_bw << std::endl; +#endif + } return true ; } @@ -134,6 +157,21 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } + _total_encoded_size_in += chunk.size ; + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_in_TS) + { + _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + + _total_encoded_size_in = 0 ; + _last_bw_estimate_in_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; +#endif + } if(!codec->decodeData(chunk,img)) { std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; @@ -144,9 +182,10 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) _decoded_output_device->showFrame(img) ; } -void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) +void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec) { std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; + _target_bandwidth_out = bytes_per_sec ; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index f5057e5e4..230c9dc9b 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -117,7 +117,7 @@ class VideoProcessor // returns the current (measured) frame rate in bytes per second. // - uint32_t currentDecodingFrameRate() const; + uint32_t currentBandwidthIn() const { return _estimated_bandwidth_in ; } private: QVideoOutputDevice *_decoded_output_device ; @@ -130,15 +130,19 @@ class VideoProcessor public: // Takes the next image to be encoded. // - bool processImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + bool processImage(const QImage& Image, uint32_t size_hint) ; bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } bool nextEncodedPacket(RsVOIPDataChunk& ) ; // Used to tweak the compression ratio so that the video can stream ok. // - void setMaximumFrameRate(uint32_t bytes_per_second) ; + void setMaximumBandwidth(uint32_t bytes_per_second) ; void setInternalFrameSize(QSize) ; + // returns the current encoding frame rate in bytes per second. + // + uint32_t currentBandwidthOut() const { return _estimated_bandwidth_out ; } + protected: std::list _encoded_out_queue ; QSize _encoded_frame_size ; @@ -152,5 +156,16 @@ class VideoProcessor FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; + + time_t _last_bw_estimate_in_TS; + time_t _last_bw_estimate_out_TS; + + uint32_t _total_encoded_size_in ; + uint32_t _total_encoded_size_out ; + + float _estimated_bandwidth_in ; + float _estimated_bandwidth_out ; + + float _target_bandwidth_out ; }; From da69db39d6a9588afa801891b3a0697b99927a2f Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 24 Aug 2015 21:42:01 -0400 Subject: [PATCH 18/23] added mutex to video processor --- plugins/VOIP/gui/VideoProcessor.cpp | 50 ++++++++++++++++++++--------- plugins/VOIP/gui/VideoProcessor.h | 4 ++- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index c4934ee0c..ef10b63a5 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -23,12 +23,13 @@ extern "C" { #include #include } -#define DEBUG_MPEG_VIDEO 1 +//#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(176,144) + :_encoded_frame_size(176,144) , vpMtx("VideoProcessor") { - _decoded_output_device = NULL ; + _decoded_output_device = NULL ; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; @@ -44,6 +45,19 @@ VideoProcessor::VideoProcessor() _last_bw_estimate_out_TS = time(NULL) ; } +VideoProcessor::~VideoProcessor() +{ + // clear encoding queue + + RS_STACK_MUTEX(vpMtx) ; + + while(!_encoded_out_queue.empty()) + { + _encoded_out_queue.back().clear() ; + _encoded_out_queue.pop_back() ; + } +} + bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { VideoCodec *codec ; @@ -68,8 +82,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) { + RS_STACK_MUTEX(vpMtx) ; _encoded_out_queue.push_back(chunk) ; - _total_encoded_size_out += chunk.size ; } @@ -77,6 +91,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) if(now > _last_bw_estimate_out_TS) { + RS_STACK_MUTEX(vpMtx) ; + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; _total_encoded_size_out = 0 ; @@ -98,6 +114,7 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) { + RS_STACK_MUTEX(vpMtx) ; if(_encoded_out_queue.empty()) return false ; @@ -157,21 +174,24 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } - _total_encoded_size_in += chunk.size ; - - time_t now = time(NULL) ; - - if(now > _last_bw_estimate_in_TS) { - _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + RS_STACK_MUTEX(vpMtx) ; + _total_encoded_size_in += chunk.size ; - _total_encoded_size_in = 0 ; - _last_bw_estimate_in_TS = now ; + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_in_TS) + { + _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + + _total_encoded_size_in = 0 ; + _last_bw_estimate_in_TS = now ; #ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; + std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; #endif - } + } + } if(!codec->decodeData(chunk,img)) { std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; @@ -794,7 +814,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) buff = decoding_buffer.data ; } - memcpy(buff,chunk.data+HEADER_SIZE,chunk.size - HEADER_SIZE) ; + memcpy(buff,&((unsigned char*)chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 230c9dc9b..a36598649 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -96,7 +96,7 @@ class VideoProcessor { public: VideoProcessor() ; - virtual ~VideoProcessor() {} + virtual ~VideoProcessor() ; enum CodecId { VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, @@ -167,5 +167,7 @@ class VideoProcessor float _estimated_bandwidth_out ; float _target_bandwidth_out ; + + RsMutex vpMtx ; }; From 68918bf9c1a6bd7841e5d2e99762607e1d918aea Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 29 Aug 2015 21:48:58 -0400 Subject: [PATCH 19/23] added variable bitrate and attempt to control it. --- plugins/VOIP/gui/AudioInputConfig.ui | 2 +- plugins/VOIP/gui/QVideoDevice.cpp | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 52 ++++++++++++++++++++++------ plugins/VOIP/gui/VideoProcessor.h | 8 ++--- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 71ed06569..62e8c98a6 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -412,7 +412,7 @@ 1 - 5.000000000000000 + 2.000000000000000 200.000000000000000 diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 516933807..3a0cc4aa9 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -77,7 +77,7 @@ void QVideoInputDevice::grabFrame() if(_video_processor != NULL) { - _video_processor->processImage(image,0) ; + _video_processor->processImage(image) ; emit networkPacketReady() ; } diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index ef10b63a5..7da37fbbe 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -36,7 +36,7 @@ VideoProcessor::VideoProcessor() _estimated_bandwidth_in = 0 ; _estimated_bandwidth_out = 0 ; - _target_bandwidth_out = 0 ; + _target_bandwidth_out = 30*1024 ; // 30 KB/s _total_encoded_size_in = 0 ; _total_encoded_size_out = 0 ; @@ -58,7 +58,7 @@ VideoProcessor::~VideoProcessor() } } -bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) +bool VideoProcessor::processImage(const QImage& img) { VideoCodec *codec ; @@ -80,7 +80,7 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { RsVOIPDataChunk chunk ; - if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) + if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),_target_bandwidth_out,chunk) && chunk.size > 0) { RS_STACK_MUTEX(vpMtx) ; _encoded_out_queue.push_back(chunk) ; @@ -318,7 +318,7 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa } -bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +bool WaveletVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) { static const int WAVELET_IMG_SIZE = 128 ; static const float W_THRESHOLD = 0.005 ; // low quality @@ -568,7 +568,28 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 200000; + encoding_context->bit_rate = 30*1024 ; // default bitrate is 30KB/s + encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 81920; + encoding_context->rc_buffer_size = 1024*1024; + encoding_context->flags |= CODEC_FLAG_PSNR; + //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; + //encoding_context->crf = 0.0f; + //encoding_context->cqp = 26; + encoding_context->i_quant_factor = 0.769f; + encoding_context->b_quant_factor = 1.4f; + encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); + encoding_context->rc_max_available_vbv_use = 0.9; + encoding_context->rc_min_vbv_overflow_use = 0.1; + encoding_context->time_base.num = 1; + encoding_context->time_base.den = 15;//framesPerSecond; + //encoding_context->me_method = ME_HEX; + encoding_context->qmin = 10; + encoding_context->qmax = 51; + encoding_context->max_qdiff = 4; + encoding_context->max_b_frames = 4; + /* resolution must be a multiple of two */ encoding_context->width = 352;//176; encoding_context->height = 288;//144; @@ -580,10 +601,9 @@ FFmpegVideo::FFmpegVideo() * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - encoding_context->gop_size = 10; + encoding_context->gop_size = 250; encoding_context->max_b_frames = 1; - //context->pix_fmt = AV_PIX_FMT_RGB24; - encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; + encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24; if (codec_id == AV_CODEC_ID_H264) av_opt_set(encoding_context->priv_data, "preset", "slow", 0); @@ -664,13 +684,22 @@ FFmpegVideo::~FFmpegVideo() free(decoding_frame_buffer); } +#define MAX_FFMPEG_ENCODING_BITRATE 81920 -bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) { #ifdef DEBUG_MPEG_VIDEO std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; #endif QImage input ; + + if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE) + { + std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; + target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; + } + encoding_context->bit_rate = target_encoding_bitrate; + encoding_context->bit_rate_tolerance = target_encoding_bitrate; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; @@ -775,6 +804,7 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } } + bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { #ifdef DEBUG_MPEG_VIDEO @@ -843,9 +873,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index a36598649..85ecae7df 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -29,7 +29,7 @@ public: JPEGVideo() ; protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; @@ -47,7 +47,7 @@ public: WaveletVideo() {} protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: @@ -73,7 +73,7 @@ public: ~FFmpegVideo() ; protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: @@ -130,7 +130,7 @@ class VideoProcessor public: // Takes the next image to be encoded. // - bool processImage(const QImage& Image, uint32_t size_hint) ; + bool processImage(const QImage& Image) ; bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } bool nextEncodedPacket(RsVOIPDataChunk& ) ; From 3d9e3e8b7cb06fd1e7c0110e5e423cbec1c019fa Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:41:38 -0400 Subject: [PATCH 20/23] attempt to setup variable bitrate. Improved codec params --- plugins/VOIP/gui/VideoProcessor.cpp | 61 +++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7da37fbbe..26fa74a53 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -26,7 +26,7 @@ extern "C" { //#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(176,144) , vpMtx("VideoProcessor") + :_encoded_frame_size(640,480) , vpMtx("VideoProcessor") { _decoded_output_device = NULL ; @@ -554,7 +554,8 @@ FFmpegVideo::FFmpegVideo() encoding_context = NULL ; //AVCodecID codec_id = AV_CODEC_ID_H264 ; - AVCodecID codec_id = AV_CODEC_ID_MPEG1VIDEO; + //AVCodecID codec_id = AV_CODEC_ID_MPEG2VIDEO; + AVCodecID codec_id = AV_CODEC_ID_MPEG4; uint8_t endcode[] = { 0, 0, 1, 0xb7 }; @@ -568,31 +569,42 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 30*1024 ; // default bitrate is 30KB/s + encoding_context->bit_rate = 10*1024 ; // default bitrate is 30KB/s encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; + +#ifdef USE_VARIABLE_BITRATE encoding_context->rc_min_rate = 0; - encoding_context->rc_max_rate = 81920; - encoding_context->rc_buffer_size = 1024*1024; + encoding_context->rc_max_rate = 10*1024;//encoding_context->bit_rate; + encoding_context->rc_buffer_size = 10*1024*1024; + encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); + encoding_context->rc_max_available_vbv_use = 1.0; + encoding_context->rc_min_vbv_overflow_use = 0.0; +#else + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 0; + encoding_context->rc_buffer_size = 0; +#endif encoding_context->flags |= CODEC_FLAG_PSNR; + encoding_context->flags |= CODEC_FLAG_TRUNCATED; + encoding_context->flags |= CODEC_CAP_PARAM_CHANGE; + encoding_context->i_quant_factor = 0.769f; + encoding_context->b_quant_factor = 1.4f; + encoding_context->time_base.num = 1; + encoding_context->time_base.den = 15;//framesPerSecond; + encoding_context->qmin = 1; + encoding_context->qmax = 51; + encoding_context->max_qdiff = 4; + + //encoding_context->me_method = ME_HEX; + //encoding_context->max_b_frames = 4; + //encoding_context->flags |= CODEC_FLAG_LOW_DELAY; // MPEG2 only //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; //encoding_context->crf = 0.0f; //encoding_context->cqp = 26; - encoding_context->i_quant_factor = 0.769f; - encoding_context->b_quant_factor = 1.4f; - encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); - encoding_context->rc_max_available_vbv_use = 0.9; - encoding_context->rc_min_vbv_overflow_use = 0.1; - encoding_context->time_base.num = 1; - encoding_context->time_base.den = 15;//framesPerSecond; - //encoding_context->me_method = ME_HEX; - encoding_context->qmin = 10; - encoding_context->qmax = 51; - encoding_context->max_qdiff = 4; - encoding_context->max_b_frames = 4; - + /* resolution must be a multiple of two */ - encoding_context->width = 352;//176; - encoding_context->height = 288;//144; + encoding_context->width = 640;//176; + encoding_context->height = 480;//144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -601,8 +613,8 @@ FFmpegVideo::FFmpegVideo() * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - encoding_context->gop_size = 250; - encoding_context->max_b_frames = 1; + encoding_context->gop_size = 100; + //encoding_context->max_b_frames = 1; encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24; if (codec_id == AV_CODEC_ID_H264) @@ -698,8 +710,9 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; } - encoding_context->bit_rate = target_encoding_bitrate; - encoding_context->bit_rate_tolerance = target_encoding_bitrate; + //encoding_context->bit_rate = target_encoding_bitrate; + encoding_context->rc_max_rate = target_encoding_bitrate; + //encoding_context->bit_rate_tolerance = target_encoding_bitrate; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; From c5cd701d4c516fa6e51887b9a83f2aaf1a35db2e Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:44:34 -0400 Subject: [PATCH 21/23] removed experimental wavelet codec --- plugins/VOIP/gui/DaubechyWavelets.h | 267 ---------------------------- plugins/VOIP/gui/VideoProcessor.cpp | 234 ------------------------ plugins/VOIP/gui/VideoProcessor.h | 21 --- 3 files changed, 522 deletions(-) delete mode 100644 plugins/VOIP/gui/DaubechyWavelets.h diff --git a/plugins/VOIP/gui/DaubechyWavelets.h b/plugins/VOIP/gui/DaubechyWavelets.h deleted file mode 100644 index c69f025b0..000000000 --- a/plugins/VOIP/gui/DaubechyWavelets.h +++ /dev/null @@ -1,267 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#ifdef USE_SSE_INSTRUCTIONS -#include -#endif - -template class DaubechyWavelets -{ - public: - typedef enum { DWT_DAUB02=2, DWT_DAUB04=4, DWT_DAUB12=12, DWT_DAUB20=20 } WaveletType ; - typedef enum { DWT_FORWARD=1, DWT_BACKWARD=0 } TransformType ; - - static void DWT2D(FLOAT *data,unsigned long int W,unsigned long int H,WaveletType type,TransformType tr) - { - unsigned long int nn[2] = {W,H} ; - wtn(&data[-1], &nn[-1],2, tr, waveletFilter(type), pwt) ; - } - static void DWT1D(FLOAT *data,unsigned long int W,WaveletType type,TransformType tr) - { - unsigned long int nn[1] = {W} ; - wtn(&data[-1], &nn[-1],1, tr, waveletFilter(type), pwt) ; - } - - - private: - class wavefilt - { - public: - wavefilt(int n) - { - int k; - FLOAT sig = -1.0; - static const FLOAT c2[5]={ 0.0, sqrt(2.0)/2.0, sqrt(2.0)/2.0, 0.0, 0.0 }; - - static const FLOAT c4[5]={ 0.0, 0.4829629131445341, 0.8365163037378079, 0.2241438680420134,-0.1294095225512604 }; - - static const FLOAT c12[13]={0.0,0.111540743350, 0.494623890398, 0.751133908021, - 0.315250351709,-0.226264693965,-0.129766867567, - 0.097501605587, 0.027522865530,-0.031582039318, - 0.000553842201, 0.004777257511,-0.001077301085}; - - static const FLOAT c20[21]={0.0,0.026670057901, 0.188176800078, 0.527201188932, - 0.688459039454, 0.281172343661,-0.249846424327, - -0.195946274377, 0.127369340336, 0.093057364604, - -0.071394147166,-0.029457536822, 0.033212674059, - 0.003606553567,-0.010733175483, 0.001395351747, - 0.001992405295,-0.000685856695,-0.000116466855, - 0.000093588670,-0.000013264203 }; - - ncof= (n==2)?4:n; - const FLOAT *tmpcc ; - cc.resize(ncof+1) ; - cr.resize(ncof+1) ; - - if (n == 2) - { - tmpcc=c2; - cc[1] = tmpcc[1] ; - cc[2] = tmpcc[2] ; - cc[3] = 0.0f ; - cc[4] = 0.0f ; - cr[1] = tmpcc[1] ; - cr[2] =-tmpcc[2] ; - cr[3] = 0.0f ; - cr[4] = 0.0f ; - - ioff = joff = -1 ; - } - else - { - if (n == 4) - tmpcc=c4; - else if (n == 12) - tmpcc=c12; - else if (n == 20) - tmpcc=c20; - else - throw std::runtime_error("unimplemented value n in pwtset"); - - for (k=1;k<=n;k++) - { - cc[k] = tmpcc[k] ; - cr[ncof+1-k]=sig*tmpcc[k]; - sig = -sig; - } - ioff = joff = -(n >> 1); - } - } - - ~wavefilt() {} - - int ncof,ioff,joff; - std::vector cc; - std::vector cr; - } ; - - static const wavefilt& waveletFilter(WaveletType type) - { - static wavefilt *daub02filt = NULL ; - static wavefilt *daub04filt = NULL ; - static wavefilt *daub12filt = NULL ; - static wavefilt *daub20filt = NULL ; - - switch(type) - { - case DWT_DAUB02: if(daub02filt == NULL) - daub02filt = new wavefilt(2) ; - return *daub02filt ; - - case DWT_DAUB04: if(daub04filt == NULL) - daub04filt = new wavefilt(4) ; - return *daub04filt ; - - case DWT_DAUB12: if(daub12filt == NULL) - daub12filt = new wavefilt(12) ; - return *daub12filt ; - - case DWT_DAUB20: if(daub20filt == NULL) - daub20filt = new wavefilt(20) ; - return *daub20filt ; - - default: - throw std::runtime_error("Unknown wavelet type.") ; - } - } - - static void pwt(FLOAT a[], unsigned long n, int isign,const wavefilt& wfilt) - { -/********************** BEGIN SIGNED PART *************************/ -/** md5sum = 2b9e1e38ac690f50806873cdb4a061ea **/ -/** Validation date = 08/10/10 **/ -/******************************************************************/ - unsigned long i,ii,ni,nj ; - - if (n < 4) - return; - - FLOAT *wksp=new FLOAT[n+1];//vector(1,n); - FLOAT ai,ai1 ; - unsigned long int nmod=wfilt.ncof*n; - unsigned long int n1=n-1; - unsigned long int nh=n >> 1; - - memset(wksp,0,(n+1)*sizeof(FLOAT)) ; - - if (isign == DWT_FORWARD) - for (ii=1,i=1;i<=n;i+=2,ii++) - { - ni=i+nmod+wfilt.ioff; - nj=i+nmod+wfilt.joff; - -#ifdef USE_SSE_INSTRUCTIONS -#warning Using SSE2 Instruction set for wavelet internal loops - for (int k=1;k<=wfilt.ncof;k+=4) - { - int jf=ni+k; - int jr=nj+k; - - sse_block w1(wfilt.cc[k],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) ; - sse_block w2(wfilt.cr[k],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) ; - - sse_block a1( a[1+((jf+0)&n1)], a[1+((jf+1)&n1)], a[1+((jf+2)&n1)], a[1+((jf+3)&n1)]) ; - sse_block a2( a[1+((jr+0)&n1)], a[1+((jr+1)&n1)], a[1+((jr+2)&n1)], a[1+((jr+3)&n1)]) ; - - sse_block wk1( w1*a1 ) ; - sse_block wk2( w2*a2 ) ; - - wksp[ii ] += wk1.sum() ; - wksp[ii+nh] += wk2.sum() ; - } -#else - for (int k=1;k<=wfilt.ncof;k++) - { - int jf=n1 & (ni+k); - int jr=n1 & (nj+k); - wksp[ii] += wfilt.cc[k]*a[jf+1]; - wksp[ii+nh] += wfilt.cr[k]*a[jr+1]; - } -#endif - } - else - for (ii=1,i=1;i<=n;i+=2,ii++) - { - ai=a[ii]; - ai1=a[ii+nh]; - ni=i+nmod+wfilt.ioff; - nj=i+nmod+wfilt.joff; - -#ifdef USE_SSE_INSTRUCTIONS - sse_block ai_sse( ai,ai,ai,ai ) ; - sse_block ai1_sse( ai1,ai1,ai1,ai1 ) ; - - for (int k=1;k<=wfilt.ncof;k+=4) - { - int jf=ni+k ; - int jr=nj+k ; // in fact we have jf==jr, so the code is simpler. - - sse_block w1(wksp[1+((jf+0) & n1)],wksp[1+((jf+1) & n1)],wksp[1+((jf+2) & n1)],wksp[1+((jf+3) & n1)]) ; - - w1 += sse_block(wfilt.cc[k+0],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) * ai_sse ; - w1 += sse_block(wfilt.cr[k+0],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) * ai1_sse ; - - wksp[1+((jr+0) & n1)] = w1[0] ; - wksp[1+((jr+1) & n1)] = w1[1] ; - wksp[1+((jr+2) & n1)] = w1[2] ; - wksp[1+((jr+3) & n1)] = w1[3] ; - } -#else - for (int k=1;k<=wfilt.ncof;++k) - { - wksp[(n1 & (ni+k))+1] += wfilt.cc[k]*ai; - wksp[(n1 & (nj+k))+1] += wfilt.cr[k]*ai1; - } -#endif - } - - for (uint j=1;j<=n;j++) - a[j]=wksp[j]; - - delete[] wksp ;//free_vector(wksp,1,n); -/********************** END SIGNED PART *************************/ - } - - static void wtn(FLOAT a[], unsigned long nn[], int ndim, int isign, const wavefilt& w,void (*wtstep)(FLOAT [], unsigned long, int,const wavefilt&)) - { - unsigned long i1,i2,i3,k,n,nnew,nprev=1,nt,ntot=1; - int idim; - FLOAT *wksp; - - for (idim=1;idim<=ndim;idim++) - ntot *= nn[idim]; - - wksp=new FLOAT[ntot+1] ; //vector(1,ntot); - - for (idim=1;idim<=ndim;idim++) - { - n=nn[idim]; - nnew=n*nprev; - - if (n > 4) - for (i2=0;i2=4;nt >>= 1) - (*wtstep)(wksp,nt,isign,w); - else - for(nt=4;nt<=n;nt <<= 1) - (*wtstep)(wksp,nt,isign,w); - - for (i3=i1+i2,k=1;k<=n;k++,i3+=nprev) a[i3]=wksp[k]; - } - - nprev=nnew; - } - delete[] wksp ;//free_vector(wksp,1,ntot); - } -}; - diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 26fa74a53..a0084ea18 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -8,7 +8,6 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" -#include "DaubechyWavelets.h" #include @@ -31,7 +30,6 @@ VideoProcessor::VideoProcessor() _decoded_output_device = NULL ; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; - //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; _estimated_bandwidth_in = 0 ; @@ -66,8 +64,6 @@ bool VideoProcessor::processImage(const QImage& img) { case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; break ; - case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; - break ; case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; break ; default: @@ -159,8 +155,6 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) { case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; break ; - case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; - break ; case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; break ; default: @@ -317,234 +311,6 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa return true ; } - -bool WaveletVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) -{ - static const int WAVELET_IMG_SIZE = 128 ; - static const float W_THRESHOLD = 0.005 ; // low quality - //static const float W_THRESHOLD = 0.0001; // high quality - //static const float W_THRESHOLD = 0.0005; // medium quality - - static const int W2 = WAVELET_IMG_SIZE ; - static const int H2 = WAVELET_IMG_SIZE ; - - assert(image.width() == W2) ; - assert(image.height() == H2) ; - - float *temp = new float[W2*H2] ; - - std::cerr << " codec type: wavelets." << std::endl; - - // We should perform some interpolation here ;-) - // - for(int i=0;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_FORWARD) ; - - // Now estimate the max energy in the W coefs, and only keep the largest. - - float mx = 0.0f ; - for(int i=0;i compressed_values ; - compressed_values.reserve(W2*H2) ; - - for(int i=0;i= W_THRESHOLD*mx) // This needs to be improved. Wavelets do not all have the same visual impact. - { - // add one value, using 16 bits for coordinates and 16 bits for the value. - - compressed_values.push_back((uint16_t)i) ; - compressed_values.push_back(quantize_16b(temp[i],mx)) ; - - //float f2 = from_quantized_16b(quantize_16b(temp[i],mx),mx) ; - - //if(fabs(f2 - temp[i]) >= 0.01*(fabs(temp[i])+fabs(f2))) - //std::cerr << " before: " << temp[i] << ", quantised=" << quantize_16b(temp[i],mx)<< ", after: " << f2 << std::endl; - } - delete[] temp ; - - // Serialise all values into a memory buffer. This needs to be taken care of because of endian issues. - - int compressed_size = 4 + compressed_values.size()*2 ; - - std::cerr << " threshold : " << W_THRESHOLD << std::endl; - std::cerr << " values kept: " << compressed_values.size()/2 << std::endl; - std::cerr << " compression: " << compressed_size/float(W2*H2*3)*100 << " %" << std::endl; - - voip_chunk.data = malloc(HEADER_SIZE + compressed_size) ; - - // build header - uint32_t flags = 0 ; - - ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO & 0xff ; - ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO >> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; - ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - - unsigned char *compressed_mem = &((unsigned char *)voip_chunk.data)[HEADER_SIZE] ; - serialise_ufloat(compressed_mem,mx) ; - - for(uint32_t i=0;i> 8 ; - } - - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - voip_chunk.size = HEADER_SIZE + compressed_size ; - - return true ; -} - -bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) -{ - static const int WAVELET_IMG_SIZE = 128 ; - - static const int W2 = WAVELET_IMG_SIZE ; - static const int H2 = WAVELET_IMG_SIZE ; - - float *temp = new float[W2*H2] ; - - const unsigned char *compressed_mem = &static_cast(chunk.data)[HEADER_SIZE] ; - int compressed_size = chunk.size - HEADER_SIZE; - - memset(temp,0,W2*H2*sizeof(float)) ; - float M = deserialise_ufloat(compressed_mem); - -#ifdef VOIP_CODEC_DEBUG - std::cerr << " codec type: wavelets." << std::endl; - std::cerr << " max coef: " << M << std::endl; -#endif - - for(int i=4;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_BACKWARD) ; - -#ifdef VOIP_CODEC_DEBUG - std::cerr << " resizing image to: " << w << "x" << h << std::endl; -#endif - - image = QImage(W2,H2,QImage::Format_RGB32) ; - - int indx = 0 ; - - for(int j=0;j1023), and p is coded on 6 bits (0->63). - // Packing [mp] into a 16bit uint16_t. M is the maximum coefficient over the quantization - // process. - // - // So this represents numbers from M * 1 * 2^{-73} to M - // - // All calculatoins are performed on x/M*2^10 - // - static const float LOG2 = log(2.0f) ; - - int m,p ; - - if(fabs(x) < 1e-8*M) - { - m = 0 ; - p = 0 ; - } - else - { - float log2f = log(fabsf(x)/M)/LOG2 ; - int mexp = (int)floor(MANTISSE_BITS - log2f) ; - - m = (int)floor(pow(2.0f,mexp+log2f)) ; - p = mexp ; - - if(p > (1<> EXPONENT_BITS ; - - if(p > 10) - return M * m / 1024.0f / (float)(1 << (p-10)) ; - else - return M * m / (float)(1 << p) ; -} - -void WaveletVideo::serialise_ufloat(unsigned char *mem, float f) -{ - if(f < 0.0f) - { - std::cerr << "(EE) Cannot serialise invalid negative float value " << f << " in " << __PRETTY_FUNCTION__ << std::endl; - return ; - } - // This serialisation is quite accurate. The max relative error is approx. - // 0.01% and most of the time less than 1e-05% The error is well distributed - // over numbers also. - // - uint32_t n = (f < 1e-7)?(~(uint32_t)0): ((uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0)))) ; - - mem[0] = n & 0xff ; n >>= 8 ; - mem[1] = n & 0xff ; n >>= 8 ; - mem[2] = n & 0xff ; n >>= 8 ; - mem[3] = n & 0xff ; -} -float WaveletVideo::deserialise_ufloat(const unsigned char *mem) -{ - uint32_t n = mem[3] ; - n = (n << 8) + mem[2] ; - n = (n << 8) + mem[1] ; - n = (n << 8) + mem[0] ; - - return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; -} - FFmpegVideo::FFmpegVideo() { // Encoding diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 85ecae7df..a3869dc5e 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -41,26 +41,6 @@ private: uint32_t _encoded_ref_frame_count ; }; -class WaveletVideo: public VideoCodec -{ -public: - WaveletVideo() {} - -protected: - virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; - virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; -private: - - static const int MANTISSE_BITS = 9 ; - static const int EXPONENT_BITS = 6 ; - - static void serialise_ufloat(unsigned char *mem, float f); - static float deserialise_ufloat(const unsigned char *mem); - - static float from_quantized_16b(uint16_t n, float M); - static uint16_t quantize_16b(float x, float M); -}; - struct AVCodec ; struct AVCodecContext ; struct AVFrame ; @@ -152,7 +132,6 @@ class VideoProcessor // ===================================================================================== JPEGVideo _jpeg_video_codec ; - WaveletVideo _ddwt_video_codec ; FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; From b1572c29243203f56a1d66b50af14abd3345c1cb Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:50:20 -0400 Subject: [PATCH 22/23] added missing deps for avcodec --- build_scripts/Debian+Ubuntu/control.precise | 2 +- build_scripts/Debian+Ubuntu/control.squeeze_bubba3 | 2 +- build_scripts/Debian+Ubuntu/control.ubuntu_lucid | 2 +- build_scripts/Debian+Ubuntu/debian/control | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_scripts/Debian+Ubuntu/control.precise b/build_scripts/Debian+Ubuntu/control.precise index b008cf5fa..70d77ea50 100644 --- a/build_scripts/Debian+Ubuntu/control.precise +++ b/build_scripts/Debian+Ubuntu/control.precise @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 index 244bdb2a0..c72507d0f 100644 --- a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 +++ b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid index 1188ca8a0..ae12f4247 100644 --- a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid +++ b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid @@ -2,7 +2,7 @@ Source: retroshare Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev, libavcodec-dev Standards-Version: 3.9.1 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/debian/control b/build_scripts/Debian+Ubuntu/debian/control index 357150da5..b251bb923 100644 --- a/build_scripts/Debian+Ubuntu/debian/control +++ b/build_scripts/Debian+Ubuntu/debian/control @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net From 77c0d562fb5c450937de0f9054c39add4f67f5ae Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 2 Sep 2015 22:14:04 -0400 Subject: [PATCH 23/23] fixed deleting/stopping video devices --- plugins/VOIP/gui/AudioInputConfig.cpp | 10 +++++++--- plugins/VOIP/gui/QVideoDevice.cpp | 14 +++++++++++++- plugins/VOIP/gui/QVideoDevice.h | 4 ++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 6ef39bce5..ea0f2e129 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -157,6 +157,8 @@ void AudioInputConfig::togglePreview(bool b) AudioInputConfig::~AudioInputConfig() { + disconnect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); + graph_source->stop() ; graph_source->setVideoInput(NULL) ; @@ -350,8 +352,10 @@ void AudioInputConfig::on_qcbTransmit_currentIndexChanged(int v) { } -void AudioInputConfig::on_Tick_timeout() { - if (!inputAudioProcessor) { +void AudioInputConfig::on_Tick_timeout() +{ + if (!inputAudioProcessor) + { inputAudioProcessor = new QtSpeex::SpeexInputProcessor(); inputAudioProcessor->open(QIODevice::WriteOnly | QIODevice::Unbuffered); @@ -376,7 +380,7 @@ void AudioInputConfig::on_Tick_timeout() { // also transmit encoded video RsVOIPDataChunk chunk ; - while(videoInput->getNextEncodedPacket(chunk)) + while((!videoInput->stopped()) && videoInput->getNextEncodedPacket(chunk)) { videoProcessor->receiveEncodedData(chunk) ; chunk.clear() ; diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 3a0cc4aa9..b81b7b79c 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -15,6 +15,11 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _echo_output_device = NULL ; } +bool QVideoInputDevice::stopped() +{ + return _timer == NULL ; +} + void QVideoInputDevice::stop() { if(_timer != NULL) @@ -54,6 +59,9 @@ void QVideoInputDevice::start() void QVideoInputDevice::grabFrame() { + if(!_timer) + return ; + IplImage *img=cvQueryFrame(_capture_device); if(img == NULL) @@ -87,6 +95,9 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { + if(!_timer) + return false ; + if(_video_processor) return _video_processor->nextEncodedPacket(chunk) ; else @@ -100,7 +111,8 @@ uint32_t QVideoInputDevice::currentBandwidth() const QVideoInputDevice::~QVideoInputDevice() { - stop() ; + stop() ; + _video_processor = NULL ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 20a837446..cc92302a2 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -51,8 +51,8 @@ class QVideoInputDevice: public QObject void start() ; void stop() ; - - protected slots: + bool stopped(); +protected slots: void grabFrame() ; signals: