ci: add datadog test logging (#44142)

* ci: add datadog test logging

(cherry picked from commit 394ea2979fda0c1551df5d5bdf7c13b57d9be181)

* ci: only upload to datadog when running tests

(cherry picked from commit c3200d8f1540f78c25673a4a9d9af3b8506f9474)

.github/workflows/pipeline-segment-electron-test.yml

@@ -146,17 +146,21 @@ jobs:
         sudo security authorizationdb write com.apple.trust-settings.admin allow
         cd src/electron
         ./script/codesign/generate-identity.sh
+    - name: Install Datadog CLI
+      run: |
+        cd src/electron
+        node script/yarn global add @datadog/datadog-ci
     - name: Run Electron Tests
       shell: bash
       env:
         MOCHA_REPORTER: mocha-multi-reporters
-        ELECTRON_TEST_RESULTS_DIR: junit
         MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap
         ELECTRON_DISABLE_SECURITY_WARNINGS: 1
         ELECTRON_SKIP_NATIVE_MODULE_TESTS: true
         DISPLAY: ':99.0'
       run: |
         cd src/electron
+        export ELECTRON_TEST_RESULTS_DIR=`pwd`/junit
         # Get which tests are on this shard
         tests_files=$(node script/split-tests ${{ matrix.shard }} ${{ inputs.target-platform == 'macos' && 2 || 3 }})
 
@@ -185,6 +189,15 @@ jobs:
             runuser -u builduser -- xvfb-run script/actions/run-tests.sh script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files
           fi
         fi
+    - name: Upload Test results to Datadog
+      env:
+        DD_ENV: ci
+        DD_SERVICE: electron
+        DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        DD_CIVISIBILITY_LOGS_ENABLED: true
+        DD_TAGS: "os.architecture:${{ inputs.target-arch }},os.family:${{ inputs.target-platform }},os.platform:${{ inputs.target-platform }},asan:${{ inputs.is-asan }}"
+      run: datadog-ci junit upload src/electron/junit/test-results-main.xml
+      if: always() && !cancelled()
     - name: Wait for active SSH sessions
       if: always() && !cancelled()
       run: |

appveyor-woa.yml

@@ -36,7 +36,7 @@ environment:
   ELECTRON_ENABLE_STACK_DUMPING: 1
   ELECTRON_ALSO_LOG_TO_STDERR: 1
   MOCHA_REPORTER: mocha-multi-reporters
-  MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap"
+  MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, mocha-junit-reporter, tap"
   DEPOT_TOOLS_WIN_TOOLCHAIN: 1
   DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_"
   GYP_MSVS_HASH_7393122652: 3ba76c5c20
@@ -257,10 +257,14 @@ for:
 
     environment:
         IGNORE_YARN_INSTALL_ERROR: 1
-        ELECTRON_TEST_RESULTS_DIR: junit
+        ELECTRON_TEST_RESULTS_DIR: C:\projects\src\electron\junit
-        MOCHA_MULTI_REPORTERS: 'mocha-junit-reporter, tap'
+        MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, mocha-junit-reporter, tap"
         MOCHA_REPORTER: mocha-multi-reporters
         ELECTRON_SKIP_NATIVE_MODULE_TESTS: true
+        DD_ENV: ci
+        DD_SERVICE: electron
+        DD_CIVISIBILITY_LOGS_ENABLED: true
+        DD_GIT_REPOSITORY_URL: "https://github.com/electron/electron.git"
 
     build_script:
       - ps: |
@@ -272,6 +276,7 @@ for:
           } else {
             $global:LASTEXITCODE = 0
           }
+      - ps: Invoke-WebRequest -Uri "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_win-x64" -OutFile "C:\projects\src\electron\datadog-ci.exe"
       - cd ..
       - mkdir out\Default
       - cd ..
@@ -327,3 +332,13 @@ for:
     on_finish:
       # Uncomment these lines to enable RDP
       # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+      - if exist electron\junit\test-results-main.xml ( appveyor-retry appveyor PushArtifact electron\junit\test-results-main.xml )
+      - ps: |
+          if ($env:RUN_TESTS -eq 'true') {
+            $env:DD_GIT_COMMIT_SHA = $env:APPVEYOR_REPO_COMMIT
+            $env:DD_GIT_BRANCH = $env:APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH
+            $env:DD_TAGS = "os.architecture:$env:TARGET_ARCH,os.family:windows,os.platform:win32"
+            if (Test-Path -Path "C:\projects\src\electron\junit\test-results-main.xml") {
+              C:\projects\src\electron\datadog-ci.exe junit upload --verbose C:\projects\src\electron\junit\test-results-main.xml
+            }
+          }

appveyor.yml

@@ -36,7 +36,7 @@ environment:
   ELECTRON_ENABLE_STACK_DUMPING: 1
   ELECTRON_ALSO_LOG_TO_STDERR: 1
   MOCHA_REPORTER: mocha-multi-reporters
-  MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap"
+  MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, mocha-junit-reporter, tap"
   DEPOT_TOOLS_WIN_TOOLCHAIN: 1
   DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_"
   GYP_MSVS_HASH_7393122652: 3ba76c5c20
@@ -247,6 +247,13 @@ for:
       only:
         - job_name: Test
 
+    environment:
+        DD_ENV: ci
+        DD_SERVICE: electron
+        DD_CIVISIBILITY_LOGS_ENABLED: true
+        DD_GIT_REPOSITORY_URL: "https://github.com/electron/electron.git"
+        ELECTRON_TEST_RESULTS_DIR: C:\projects\src\electron\junit
+
     init:
       - ps: |
           if ($env:RUN_TESTS -ne 'true') {
@@ -262,6 +269,7 @@ for:
           } else {
             $global:LASTEXITCODE = 0
           }
+      - npm install -g @datadog/datadog-ci
       - cd ..
       - mkdir out\Default
       - cd ..
@@ -319,3 +327,13 @@ for:
     on_finish:
       # Uncomment these lines to enable RDP
       # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+      - if exist electron\junit\test-results-main.xml ( appveyor-retry appveyor PushArtifact electron\junit\test-results-main.xml )
+      - ps: |
+          if ($env:RUN_TESTS -eq 'true') {
+            $env:DD_GIT_COMMIT_SHA = $env:APPVEYOR_REPO_COMMIT
+            $env:DD_GIT_BRANCH = $env:APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH
+            $env:DD_TAGS = "os.architecture:$env:TARGET_ARCH,os.family:windows,os.platform:win32"
+            if (Test-Path -Path "C:\projects\src\electron\junit\test-results-main.xml") {
+              C:\Users\appveyor\AppData\Roaming\npm\datadog-ci.ps1 junit upload --verbose C:\projects\src\electron\junit\test-results-main.xml
+            }
+          }