[host] dxgi: add helper to manage COM object memory

host/platform/Windows/capture/DXGI/CMakeLists.txt

@@ -7,6 +7,7 @@ add_library(capture_DXGI STATIC
   src/d3d12.c
   src/ods_capture.c
   src/util.c
+  src/com_ref.c
 )
 
 add_definitions("-DCOBJMACROS -DINITGUID")

host/platform/Windows/capture/DXGI/src/com_ref.c

@@ -0,0 +1,137 @@
+/**
+ * Looking Glass
+ * Copyright © 2017-2023 The Looking Glass Authors
+ * https://looking-glass.io
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#define COMREF_INTERNAL
+#include "com_ref.h"
+
+#include "common/debug.h"
+#include "common/vector.h"
+
+typedef struct
+{
+  int          scope;
+  IUnknown *   value;
+  IUnknown *** ref;
+}
+COMRef;
+
+static int    comScope         = -1;
+static Vector comObjectsLocal  = {0};
+static Vector comObjectsGlobal = {0};
+
+bool comRef_init(unsigned globals, unsigned locals)
+{
+  if (!vector_create(&comObjectsGlobal, sizeof(COMRef), globals))
+    return false;
+
+  if (!vector_create(&comObjectsLocal, sizeof(COMRef), locals))
+  {
+    vector_destroy(&comObjectsGlobal);
+    return false;
+  }
+
+  return true;
+}
+
+void comRef_free(void)
+{
+  COMRef * ref;
+
+  if (comScope > -1)
+  {
+    DEBUG_WARN("There is %d unmatched `comRef_scopePush` calls", comScope+1);
+    vector_forEachRef(ref, &comObjectsLocal)
+      if (ref->value)
+        IUnknown_Release(ref->value);
+  }
+
+  vector_forEachRef(ref, &comObjectsGlobal)
+  {
+    if (ref->ref)
+      *ref->ref = NULL;
+
+    if (ref->value)
+      IUnknown_Release(ref->value);
+  }
+
+  comScope = -1;
+  vector_destroy(&comObjectsLocal);
+  vector_destroy(&comObjectsGlobal);
+}
+
+static IUnknown ** comRef_new(Vector * vector, IUnknown *** dst)
+{
+  // we must not allow the vector to grow as if the realloc moves to a new
+  // address it will invalidate any external pointers to members in it
+  DEBUG_ASSERT(vector_size(vector) < vector_capacity(vector) &&
+    "comRef vector too small!");
+
+  COMRef * ref = (COMRef *)vector_push(vector, NULL);
+  if (!ref)
+  {
+    DEBUG_ERROR("Failed to allocate ram for com object");
+    return NULL;
+  }
+
+  ref->scope = comScope;
+  ref->ref   = dst;
+  ref->value = NULL;
+
+  if (dst)
+    *dst = &ref->value;
+
+  return &ref->value;
+}
+
+IUnknown ** comRef_newGlobal(IUnknown *** dst)
+{
+  return comRef_new(&comObjectsGlobal, dst);
+}
+
+IUnknown ** comRef_newLocal(IUnknown *** dst)
+{
+  IUnknown ** ret = comRef_new(&comObjectsLocal, NULL);
+  *dst = ret;
+  return ret;
+}
+
+void comRef_scopePush(void) { ++comScope; }
+
+void comRef_scopePop(void)
+{
+  DEBUG_ASSERT(comScope >= 0);
+
+  COMRef * ref;
+  while(vector_size(&comObjectsLocal) > 0)
+  {
+    ref = (COMRef *)vector_ptrTo(&comObjectsLocal,
+      vector_size(&comObjectsLocal) - 1);
+
+    if (ref->scope < comScope)
+      break;
+
+    if (ref->value)
+      IUnknown_Release(ref->value);
+
+    vector_pop(&comObjectsLocal);
+  }
+
+  --comScope;
+}

host/platform/Windows/capture/DXGI/src/com_ref.h

@@ -0,0 +1,96 @@
+/**
+ * Looking Glass
+ * Copyright © 2017-2023 The Looking Glass Authors
+ * https://looking-glass.io
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdbool.h>
+#include <windows.h>
+
+/**
+ * These functions are to assist in tracking and relasing COM objects
+ */
+
+/**
+ * Initialize the com object tracking
+ */
+bool comRef_init(unsigned globals, unsigned locals);
+
+/**
+ * Release globals and deinitialize the com object tracking
+ */
+void comRef_free(void);
+
+/**
+ * Create a new global COM reference
+ */
+IUnknown ** comRef_newGlobal(IUnknown *** dst);
+
+/**
+ * Create a new locally scoped COM reference
+ */
+IUnknown ** comRef_newLocal(IUnknown *** dst);
+
+/**
+ * Define and create a new locally scoped COM reference
+ */
+#define comRef_defineLocal(type, name) \
+  type ** name; \
+  comRef_newLocal(&name);
+
+/**
+ * Release a COM reference immediately
+ * This is just a helper, the ref is still tracked if used again
+ */
+inline static ULONG comRef_release(IUnknown ** ref)
+{
+  ULONG count = 0;
+  if (*ref)
+    count = IUnknown_Release(*ref);
+  *ref = NULL;
+  return count;
+}
+
+/**
+ * Create a new local scope
+ */
+void comRef_scopePush(void);
+
+/**
+ * Exit from a local scope and release all locals
+ */
+void comRef_scopePop (void);
+
+/**
+ * Macros to prevent needing to typecast calls to these methods
+ */
+#ifndef COMREF_INTERNAL
+  #define comRef_newGlobal(dst) comRef_newGlobal((IUnknown ***)(dst))
+  #define comRef_newLocal(dst)  comRef_newLocal((IUnknown ***)(dst))
+  #define comRef_release(ref)   comRef_release((IUnknown **)(ref))
+#endif
+
+/**
+ * Convert a local to a global
+ */
+#define comRef_toGlobal(dst, src) \
+{ \
+  IUnknown ** global = comRef_newGlobal(&(dst)); \
+  DEBUG_ASSERT(global && "comRef_newGlobal failed\n"); \
+  *global = (IUnknown*)*(src); \
+  *(src)  = NULL; \
+}