[lldb][Windows] Disable debug heap by default (#212126)

This disables Windows' default debug heap when launching a process.

The motivation is similar to what Microsoft wrote on their [blog post
for Visual Studio
2015](https://devblogs.microsoft.com/cppblog/c-debugging-improvements-in-visual-studio-14/)
where they disabled it: The debug heap comes with a performance cost and
the [C runtime already does some heap
checking](https://learn.microsoft.com/en-us/cpp/c-runtime-library/crt-debug-heap-details?view=msvc-170).

Towards #201690. Notably, lldb-dap won't set this when launching in a
terminal. I think we'd want an option similar to `disableASLR` for
lldb-dap.

lldb/include/lldb/Utility/Environment.h

@@ -44,6 +44,7 @@ public:
 
   using Base::begin;
   using Base::clear;
+  using Base::contains;
   using Base::count;
   using Base::empty;
   using Base::end;

lldb/source/Plugins/Platform/Windows/CMakeLists.txt

@@ -1,3 +1,15 @@
+lldb_tablegen(PlatformWindowsProperties.inc -gen-lldb-property-defs
+  SOURCE PlatformWindowsProperties.td
+  TARGET LLDBPluginPlatformWindowsPropertiesGen)
+
+lldb_tablegen(PlatformWindowsPropertiesEnum.inc -gen-lldb-property-enum-defs
+  SOURCE PlatformWindowsProperties.td
+  TARGET LLDBPluginPlatformWindowsPropertiesEnumGen)
+
+lldb_tablegen(PlatformWindowsProperties.json -dump-json
+  SOURCE PlatformWindowsProperties.td
+  TARGET LLDBPluginPlatformWindowsPropertiesJsonGen)
+
 add_lldb_library(lldbPluginPlatformWindows PLUGIN
   PlatformWindows.cpp
 
@@ -10,3 +22,7 @@ add_lldb_library(lldbPluginPlatformWindows PLUGIN
     lldbTarget
     lldbPluginPlatformGDB
   )
+
+add_dependencies(lldbPluginPlatformWindows
+  LLDBPluginPlatformWindowsPropertiesGen
+  LLDBPluginPlatformWindowsPropertiesEnumGen)

lldb/source/Plugins/Platform/Windows/PlatformWindows.cpp

@@ -45,6 +45,35 @@ LLDB_PLUGIN_DEFINE(PlatformWindows)
 
 static uint32_t g_initialize_count = 0;
 
+namespace {
+
+#define LLDB_PROPERTIES_windows
+#include "PlatformWindowsProperties.inc"
+
+enum {
+#define LLDB_PROPERTIES_windows
+#include "PlatformWindowsPropertiesEnum.inc"
+};
+
+class PluginProperties : public Properties {
+public:
+  PluginProperties() {
+    m_collection_sp = std::make_shared<OptionValueProperties>("windows");
+    m_collection_sp->Initialize(g_windows_properties_def);
+  }
+
+  bool DisableDebugHeap() const {
+    return GetPropertyAtIndexAs<bool>(ePropertyDisableDebugHeap, true);
+  }
+};
+
+static PluginProperties &GetGlobalProperties() {
+  static PluginProperties g_settings;
+  return g_settings;
+}
+
+} // end of anonymous namespace
+
 PlatformSP PlatformWindows::CreateInstance(bool force,
                                            const lldb_private::ArchSpec *arch) {
   // The only time we create an instance is when we are creating a remote
@@ -92,6 +121,15 @@ llvm::StringRef PlatformWindows::GetPluginDescriptionStatic(bool is_host) {
                  : "Remote Windows user platform plug-in.";
 }
 
+void PlatformWindows::DebuggerInitialize(Debugger &debugger) {
+  if (!PluginManager::GetSettingForPlatformPlugin(debugger, "windows")) {
+    PluginManager::CreateSettingForPlatformPlugin(
+        debugger, GetGlobalProperties().GetValueProperties(),
+        "Properties for the Windows platform plugin.",
+        /*is_global_property=*/true);
+  }
+}
+
 void PlatformWindows::Initialize() {
   Platform::Initialize();
 
@@ -105,7 +143,7 @@ void PlatformWindows::Initialize() {
     PluginManager::RegisterPlugin(
         PlatformWindows::GetPluginNameStatic(false),
         PlatformWindows::GetPluginDescriptionStatic(false),
-        PlatformWindows::CreateInstance);
+        PlatformWindows::CreateInstance, PlatformWindows::DebuggerInitialize);
   }
 }
 
@@ -559,6 +597,12 @@ ProcessSP PlatformWindows::DebugProcess(ProcessLaunchInfo &launch_info,
     return Attach(attach_info, debugger, &target, error);
   }
 
+  Environment &env = launch_info.GetEnvironment();
+  if (GetGlobalProperties().DisableDebugHeap() &&
+      !env.contains("_NO_DEBUG_HEAP")) {
+    env.try_emplace("_NO_DEBUG_HEAP", "1");
+  }
+
   ProcessSP process_sp =
       target.CreateProcess(launch_info.GetListener(),
                            launch_info.GetProcessPluginName(), nullptr, false);

lldb/source/Plugins/Platform/Windows/PlatformWindows.h

@@ -31,6 +31,8 @@ public:
 
   static llvm::StringRef GetPluginDescriptionStatic(bool is_host);
 
+  static void DebuggerInitialize(Debugger &debugger);
+
   llvm::StringRef GetPluginName() override {
     return GetPluginNameStatic(IsHost());
   }

lldb/source/Plugins/Platform/Windows/PlatformWindowsProperties.td

@@ -0,0 +1,10 @@
+include "../../../../include/lldb/Core/PropertiesBase.td"
+
+let Definition = "windows", Path = "platform.plugin.windows" in {
+  def DisableDebugHeap: Property<"disable-debug-heap", "Boolean">,
+    Global,
+    DefaultTrue,
+    Desc<"Specifies that the debug heap should not be used for debugging. "
+    "The Windows debug heap has additional checks for heap related bugs at the cost of a noticeable performance penalty. "
+    "By enabling this setting, _NO_DEBUG_HEAP=1 will be added to the environment.">;
+}

lldb/test/API/windows/debug-heap/Makefile

@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules

lldb/test/API/windows/debug-heap/TestWindowsDebugHeap.py

@@ -0,0 +1,47 @@
+"""
+Test that LLDB disables the debug heap on Windows.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from typing import List
+
+
+@skipUnlessWindows
+class DebugHeapTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def tearDown(self):
+        self.runCmd("settings clear platform.plugin.windows.disable-debug-heap")
+        return super().tearDown()
+
+    def _run_to_exit(self, envp: List[str] = []):
+        self.build()
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+        self.assertTrue(target, VALID_TARGET)
+        self.dbg.SetAsync(False)
+        process = target.LaunchSimple([], envp, self.get_process_working_directory())
+        self.assertTrue(process and process.IsValid(), PROCESS_IS_VALID)
+        self.assertState(process.GetState(), lldb.eStateExited)
+        return process.GetSTDOUT(256)
+
+    def test_default(self):
+        output = self._run_to_exit()
+        self.assertIn("_NO_DEBUG_HEAP=1", output)
+        output = self._run_to_exit(["_NO_DEBUG_HEAP=2"])
+        self.assertIn("_NO_DEBUG_HEAP=2", output)
+
+    def test_disabled(self):
+        self.runCmd("settings set platform.plugin.windows.disable-debug-heap false")
+        output = self._run_to_exit()
+        self.assertNotIn("_NO_DEBUG_HEAP", output)
+        output = self._run_to_exit(["_NO_DEBUG_HEAP=2"])
+        self.assertIn("_NO_DEBUG_HEAP=2", output)
+
+    def test_enabled(self):
+        self.runCmd("settings set platform.plugin.windows.disable-debug-heap true")
+        output = self._run_to_exit()
+        self.assertIn("_NO_DEBUG_HEAP=1", output)
+        output = self._run_to_exit(["_NO_DEBUG_HEAP=2"])
+        self.assertIn("_NO_DEBUG_HEAP=2", output)

lldb/test/API/windows/debug-heap/main.c

@@ -0,0 +1,10 @@
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char **argv, char **envp) {
+  for (char **p = envp; *p; ++p) {
+    if (strstr(*p, "_NO_DEBUG_HEAP=") == *p)
+      printf("%s\n", *p);
+  }
+  return 0;
+}