From 1de1ca906a36adfde432190284f8c378118a3c94 Mon Sep 17 00:00:00 2001 From: Heilig Benedek Date: Wed, 18 Apr 2018 02:45:15 +0200 Subject: [PATCH] Reimplement FrameSubscriber with mojo VideoCapture APIs --- atom/browser/api/atom_api_web_contents.cc | 10 +- atom/browser/api/atom_api_web_contents.h | 2 + atom/browser/api/frame_subscriber.cc | 136 +++++++++--------- atom/browser/api/frame_subscriber.h | 43 +++--- .../osr/osr_render_widget_host_view.cc | 9 -- .../browser/osr/osr_render_widget_host_view.h | 3 - 6 files changed, 99 insertions(+), 104 deletions(-) diff --git a/atom/browser/api/atom_api_web_contents.cc b/atom/browser/api/atom_api_web_contents.cc index 1a2f33bd43d0..0326ead6152c 100644 --- a/atom/browser/api/atom_api_web_contents.cc +++ b/atom/browser/api/atom_api_web_contents.cc @@ -1637,10 +1637,8 @@ void WebContents::SendInputEvent(v8::Isolate* isolate, } void WebContents::BeginFrameSubscription(mate::Arguments* args) { - bool only_dirty = false; FrameSubscriber::FrameCaptureCallback callback; - args->GetNext(&only_dirty); if (!args->GetNext(&callback)) { args->ThrowError(); return; @@ -1648,16 +1646,12 @@ void WebContents::BeginFrameSubscription(mate::Arguments* args) { auto* const view = web_contents()->GetRenderWidgetHostView(); if (view) { - std::unique_ptr frame_subscriber( - new FrameSubscriber(isolate(), view, callback, only_dirty)); - view->BeginFrameSubscription(std::move(frame_subscriber)); + frame_subscriber_.reset(new FrameSubscriber(isolate(), view, callback)); } } void WebContents::EndFrameSubscription() { - auto* const view = web_contents()->GetRenderWidgetHostView(); - if (view) - view->EndFrameSubscription(); + frame_subscriber_.reset(); } void WebContents::StartDrag(const mate::Dictionary& item, diff --git a/atom/browser/api/atom_api_web_contents.h b/atom/browser/api/atom_api_web_contents.h index 31e04ee3e1ec..23de06cbddeb 100644 --- a/atom/browser/api/atom_api_web_contents.h +++ b/atom/browser/api/atom_api_web_contents.h @@ -438,6 +438,8 @@ class WebContents : public mate::TrackableObject, std::unique_ptr dialog_manager_; std::unique_ptr guest_delegate_; + std::unique_ptr frame_subscriber_; + // The host webcontents that may contain this webcontents. WebContents* embedder_ = nullptr; diff --git a/atom/browser/api/frame_subscriber.cc b/atom/browser/api/frame_subscriber.cc index 8c2ce34a8a05..5bbd18571d2e 100644 --- a/atom/browser/api/frame_subscriber.cc +++ b/atom/browser/api/frame_subscriber.cc @@ -5,10 +5,9 @@ #include "atom/browser/api/frame_subscriber.h" #include "atom/common/native_mate_converters/gfx_converter.h" -#include "base/bind.h" -#include "content/public/browser/render_widget_host.h" -#include "ui/display/display.h" -#include "ui/display/screen.h" +#include "components/viz/host/host_frame_sink_manager.h" +#include "content/browser/compositor/surface_utils.h" +#include "content/browser/renderer_host/render_widget_host_view_base.h" #include "atom/common/node_includes.h" @@ -18,91 +17,96 @@ namespace api { FrameSubscriber::FrameSubscriber(v8::Isolate* isolate, content::RenderWidgetHostView* view, - const FrameCaptureCallback& callback, - bool only_dirty) + const FrameCaptureCallback& callback) : isolate_(isolate), view_(view), callback_(callback), - only_dirty_(only_dirty), - source_id_for_copy_request_(base::UnguessableToken::Create()), - weak_factory_(this) {} + video_consumer_binding_(this), + weak_factory_(this) { + video_capturer_ = CreateVideoCapturer(); + video_capturer_->SetResolutionConstraints( + view_->GetViewBounds().size(), + view_->GetViewBounds().size(), true); + video_capturer_->SetFormat(media::PIXEL_FORMAT_ARGB, + media::COLOR_SPACE_UNSPECIFIED); + + viz::mojom::FrameSinkVideoConsumerPtr consumer; + video_consumer_binding_.Bind(mojo::MakeRequest(&consumer)); + video_capturer_->Start(std::move(consumer)); +} FrameSubscriber::~FrameSubscriber() = default; -bool FrameSubscriber::ShouldCaptureFrame( - const gfx::Rect& dirty_rect, - base::TimeTicks present_time, - scoped_refptr* storage, - DeliverFrameCallback* callback) { - if (!view_) - return false; - - if (dirty_rect.IsEmpty()) - return false; - - gfx::Rect rect = gfx::Rect(view_->GetVisibleViewportSize()); - if (only_dirty_) - rect = dirty_rect; - - gfx::Size view_size = rect.size(); - gfx::Size bitmap_size = view_size; - gfx::NativeView native_view = view_->GetNativeView(); - const float scale = display::Screen::GetScreen() - ->GetDisplayNearestView(native_view) - .device_scale_factor(); - if (scale > 1.0f) - bitmap_size = gfx::ScaleToCeiledSize(view_size, scale); - - rect = gfx::Rect(rect.origin(), bitmap_size); - - view_->CopyFromSurface( - rect, rect.size(), - base::Bind(&FrameSubscriber::OnFrameDelivered, weak_factory_.GetWeakPtr(), - callback_, rect), - kBGRA_8888_SkColorType); - - return false; -} - -const base::UnguessableToken& FrameSubscriber::GetSourceIdForCopyRequest() { - return source_id_for_copy_request_; -} - -void FrameSubscriber::OnFrameDelivered(const FrameCaptureCallback& callback, - const gfx::Rect& damage_rect, - const SkBitmap& bitmap, - content::ReadbackResponse response) { - if (response != content::ReadbackResponse::READBACK_SUCCESS) - return; - +void FrameSubscriber::OnFrameCaptured( + mojo::ScopedSharedBufferHandle buffer, + uint32_t buffer_size, + ::media::mojom::VideoFrameInfoPtr info, + const gfx::Rect& update_rect, + const gfx::Rect& content_rect, + viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr callbacks) { v8::Locker locker(isolate_); v8::HandleScope handle_scope(isolate_); - size_t rgb_row_size = bitmap.width() * bitmap.bytesPerPixel(); - - v8::MaybeLocal buffer = - node::Buffer::New(isolate_, rgb_row_size * bitmap.height()); - - if (buffer.IsEmpty()) + gfx::Size view_size = view_->GetViewBounds().size(); + if (view_size != content_rect.size()) { + video_capturer_->SetResolutionConstraints(view_size, view_size, true); + video_capturer_->RequestRefreshFrame(); return; + } - auto local_buffer = buffer.ToLocalChecked(); + if (!buffer.is_valid()) { + callbacks->Done(); + return; + } + + mojo::ScopedSharedBufferMapping mapping = buffer->Map(buffer_size); + if (!mapping) { + return; + } + + SkImageInfo image_info = SkImageInfo::MakeN32( + content_rect.width(), content_rect.height(), kPremul_SkAlphaType); + SkPixmap pixmap(image_info, mapping.get(), + media::VideoFrame::RowBytes(media::VideoFrame::kARGBPlane, + info->pixel_format, + info->coded_size.width())); + frame_.installPixels(pixmap); + size_t rgb_row_size = frame_.width() * frame_.bytesPerPixel(); + v8::MaybeLocal node_buffer = + node::Buffer::New(isolate_, rgb_row_size * frame_.height()); + auto local_buffer = node_buffer.ToLocalChecked(); { - auto* source = static_cast(bitmap.getPixels()); + auto* source = static_cast(frame_.getPixels()); auto* target = node::Buffer::Data(local_buffer); - for (int y = 0; y < bitmap.height(); ++y) { + for (int y = 0; y < frame_.height(); ++y) { memcpy(target, source, rgb_row_size); - source += bitmap.rowBytes(); + source += frame_.rowBytes(); target += rgb_row_size; } } v8::Local damage = - mate::Converter::ToV8(isolate_, damage_rect); + mate::Converter::ToV8(isolate_, update_rect); callback_.Run(local_buffer, damage); + + shared_memory_mapping_ = std::move(mapping); + shared_memory_releaser_ = std::move(callbacks); +} + +void FrameSubscriber::OnTargetLost(const viz::FrameSinkId& frame_sink_id) {} + +void FrameSubscriber::OnStopped() {} + +viz::mojom::FrameSinkVideoCapturerPtr FrameSubscriber::CreateVideoCapturer() { + auto* view_base = static_cast(view_); + viz::mojom::FrameSinkVideoCapturerPtr video_capturer; + content::GetHostFrameSinkManager()->CreateVideoCapturer( + mojo::MakeRequest(&video_capturer)); + video_capturer->ChangeTarget(view_base->GetFrameSinkId()); + return video_capturer; } } // namespace api diff --git a/atom/browser/api/frame_subscriber.h b/atom/browser/api/frame_subscriber.h index 4dbef58d9f0b..5e9df88352c8 100644 --- a/atom/browser/api/frame_subscriber.h +++ b/atom/browser/api/frame_subscriber.h @@ -7,46 +7,53 @@ #include "base/callback.h" #include "base/memory/weak_ptr.h" -#include "content/browser/renderer_host/render_widget_host_view_frame_subscriber.h" -#include "content/public/browser/readback_types.h" #include "content/public/browser/render_widget_host_view.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/viz/privileged/interfaces/compositing/frame_sink_video_capture.mojom.h" #include "third_party/skia/include/core/SkBitmap.h" -#include "ui/gfx/geometry/size.h" #include "v8/include/v8.h" namespace atom { namespace api { -class FrameSubscriber : public content::RenderWidgetHostViewFrameSubscriber { +class FrameSubscriber : public viz::mojom::FrameSinkVideoConsumer { public: using FrameCaptureCallback = base::Callback, v8::Local)>; FrameSubscriber(v8::Isolate* isolate, content::RenderWidgetHostView* view, - const FrameCaptureCallback& callback, - bool only_dirty); + const FrameCaptureCallback& callback); ~FrameSubscriber() override; - bool ShouldCaptureFrame(const gfx::Rect& damage_rect, - base::TimeTicks present_time, - scoped_refptr* storage, - DeliverFrameCallback* callback) override; - const base::UnguessableToken& GetSourceIdForCopyRequest() override; - private: - void OnFrameDelivered(const FrameCaptureCallback& callback, - const gfx::Rect& damage_rect, - const SkBitmap& bitmap, - content::ReadbackResponse response); + void OnFrameCaptured( + mojo::ScopedSharedBufferHandle buffer, + uint32_t buffer_size, + ::media::mojom::VideoFrameInfoPtr info, + const gfx::Rect& update_rect, + const gfx::Rect& content_rect, + viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr callbacks) override; + void OnTargetLost(const viz::FrameSinkId& frame_sink_id) override; + void OnStopped() override; + viz::mojom::FrameSinkVideoCapturerPtr CreateVideoCapturer(); v8::Isolate* isolate_; content::RenderWidgetHostView* view_; FrameCaptureCallback callback_; - bool only_dirty_; - base::UnguessableToken source_id_for_copy_request_; + SkBitmap frame_; + + // This object keeps the shared memory that backs |frame_| mapped. + mojo::ScopedSharedBufferMapping shared_memory_mapping_; + + // This object prevents FrameSinkVideoCapturer from recycling the shared + // memory that backs |frame_|. + viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr shared_memory_releaser_; + + viz::mojom::FrameSinkVideoCapturerPtr video_capturer_; + mojo::Binding video_consumer_binding_; base::WeakPtrFactory weak_factory_; diff --git a/atom/browser/osr/osr_render_widget_host_view.cc b/atom/browser/osr/osr_render_widget_host_view.cc index b2b2cec74681..27a0d8fc4749 100644 --- a/atom/browser/osr/osr_render_widget_host_view.cc +++ b/atom/browser/osr/osr_render_widget_host_view.cc @@ -661,15 +661,6 @@ void OffScreenRenderWidgetHostView::CopyFromSurfaceToVideoFrame( src_subrect, target, callback); } -void OffScreenRenderWidgetHostView::BeginFrameSubscription( - std::unique_ptr subscriber) { - GetDelegatedFrameHost()->BeginFrameSubscription(std::move(subscriber)); -} - -void OffScreenRenderWidgetHostView::EndFrameSubscription() { - GetDelegatedFrameHost()->EndFrameSubscription(); -} - void OffScreenRenderWidgetHostView::InitAsGuest( content::RenderWidgetHostView* parent_host_view, content::RenderWidgetHostViewGuest* guest_view) { diff --git a/atom/browser/osr/osr_render_widget_host_view.h b/atom/browser/osr/osr_render_widget_host_view.h index bba0c25c2a33..3413c0832215 100644 --- a/atom/browser/osr/osr_render_widget_host_view.h +++ b/atom/browser/osr/osr_render_widget_host_view.h @@ -151,9 +151,6 @@ class OffScreenRenderWidgetHostView const gfx::Rect& src_subrect, scoped_refptr target, const base::Callback& callback) override; - void BeginFrameSubscription( - std::unique_ptr) override; - void EndFrameSubscription() override; void InitAsGuest(content::RenderWidgetHostView*, content::RenderWidgetHostViewGuest*) override; bool HasAcceleratedSurface(const gfx::Size&) override;