[host] nvfbc: force composition to capture some full screen apps

NvFBC is unable to capture certain applications that bypasses the DWM
compositor, for example, Firefox playing video in full screen. This
has been a known issue for a long time with Nvidia's ShadowPlay, see:
* https://www.nvidia.com/en-us/geforce/forums/geforce-experience/14/233709/
* https://crbug.com/609857

Nvidia won't fix this, but there are workarounds. For example, we
create a transparent 1x1 layered window, which forces desktop composition
to be enabled.

Note that SetLayeredWindowAttributes also supports alpha-based transparency,
but setting transparency to 0 will cause DWM to skip composition. We could
use a transparency of 1, but this ruins the image by the slightest bit,
which is unacceptable. Therefore, we must use chroma key-based
transparency, which tricks DWM into compositing despite being fully
transparent.
This commit is contained in:
Quantum 2021-01-20 02:11:49 -05:00 committed by Geoffrey McRae
parent 428b498cca
commit ffa72c7992
4 changed files with 165 additions and 0 deletions

View file

@ -9,6 +9,7 @@ add_library(platform_Windows STATIC
src/platform.c
src/service.c
src/mousehook.c
src/force_compose.c
)
# allow use of functions for Windows Vista or later

View file

@ -21,6 +21,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "interface/platform.h"
#include "common/windebug.h"
#include "windows/mousehook.h"
#include "windows/force_compose.h"
#include "common/option.h"
#include "common/framebuffer.h"
#include "common/event.h"
@ -191,6 +192,8 @@ static bool nvfbc_init(void)
if (this->seperateCursor)
this->cursorEvents[1] = lgWrapEvent(event);
dwmForceComposition();
DEBUG_INFO("Cursor mode : %s", this->seperateCursor ? "decoupled" : "integrated");
Sleep(100);
@ -221,6 +224,7 @@ static void nvfbc_stop(void)
static bool nvfbc_deinit(void)
{
mouseHook_remove();
dwmUnforceComposition();
if (this->cursorEvents[0])
{

View file

@ -0,0 +1,21 @@
/*
Looking Glass - KVM FrameRelay (KVMFR) Client
Copyright (C) 2021 Guanzhong Chen <quantum2048@gmail.com>
https://looking-glass.io
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
*/
void dwmForceComposition(void);
void dwmUnforceComposition(void);

View file

@ -0,0 +1,139 @@
/*
Looking Glass - KVM FrameRelay (KVMFR) Client
Copyright (C) 2021 Guanzhong Chen <quantum2048@gmail.com>
https://looking-glass.io
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include "windows/force_compose.h"
#include "common/windebug.h"
#include <windows.h>
#include <stdbool.h>
struct ForceCompose
{
HANDLE event;
HANDLE thread;
};
static struct ForceCompose forceCompose = { 0 };
LRESULT CALLBACK windowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
switch (message)
{
case WM_CLOSE:
return 0; // Don't allow close
case WM_DESTROY:
PostQuitMessage(0);
return 0;
case WM_PAINT:
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
FillRect(hdc, &ps.rcPaint, (HBRUSH) (COLOR_WINDOW+1));
EndPaint(hwnd, &ps);
return 0;
}
}
return DefWindowProc(hwnd, message, wParam, lParam);
}
static DWORD WINAPI threadProc(LPVOID lParam)
{
WNDCLASSA wc = { 0 };
wc.lpfnWndProc = windowProc;
wc.hInstance = (HINSTANCE) GetModuleHandle(NULL);
wc.lpszClassName = "looking-glass-force-composition";
RegisterClass(&wc);
HWND hwnd = CreateWindowEx(
WS_EX_TOOLWINDOW | WS_EX_TOPMOST | WS_EX_LAYERED, wc.lpszClassName,
"Looking Glass Helper Window", WS_POPUP, 0, 0, 1, 1, NULL, NULL,
wc.hInstance, NULL
);
if (!hwnd)
{
DEBUG_ERROR("Failed to create window to force composition");
goto exit;
}
SetLayeredWindowAttributes(hwnd, GetSysColor(COLOR_WINDOW), 0, LWA_COLORKEY);
ShowWindow(hwnd, SW_SHOW);
DEBUG_INFO("Created window to force composition");
MSG msg;
while (true)
{
switch (MsgWaitForMultipleObjects(1, &forceCompose.event, FALSE, INFINITE, QS_ALLINPUT))
{
case WAIT_OBJECT_0:
DEBUG_INFO("Force composition received quit request");
DestroyWindow(hwnd);
// Do not wait on the event after it has been signaled.
while (GetMessage(&msg, NULL, 0, 0) > 0)
{
TranslateMessage(&msg);
DispatchMessage(&msg);
}
goto exit;
case WAIT_OBJECT_0 + 1:
while (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE))
{
if (msg.message == WM_QUIT)
goto exit;
TranslateMessage(&msg);
DispatchMessage(&msg);
}
break;
default:
goto exit;
}
}
exit:
UnregisterClass(wc.lpszClassName, wc.hInstance);
return 0;
}
void dwmForceComposition(void)
{
if (!forceCompose.event)
{
forceCompose.event = CreateEventA(NULL, FALSE, FALSE, NULL);
if (!forceCompose.event)
{
DEBUG_WINERROR("Failed to create unforce composition event", GetLastError());
return;
}
}
forceCompose.thread = CreateThread(NULL, 0, threadProc, NULL, 0, NULL);
}
void dwmUnforceComposition(void)
{
if (!forceCompose.event)
return;
SetEvent(forceCompose.event);
WaitForSingleObject(forceCompose.thread, INFINITE);
CloseHandle(forceCompose.thread);
forceCompose.thread = NULL;
}