From 4b905ae2bdb04e4cb9779792384ab2acf01ef75b Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Wed, 27 Apr 2016 17:28:44 -0700 Subject: [PATCH 1/9] Reducing Telemetry performance delay Added telemetry timeout Added SHA256 hashed arguments Added Continous Integration flag Added variable rate sampling Removed ExitCode from telemetry --- src/dotnet/Program.cs | 9 +-- src/dotnet/Telemetry.cs | 123 ++++++++++++++++++++++++++++++---------- 2 files changed, 95 insertions(+), 37 deletions(-) diff --git a/src/dotnet/Program.cs b/src/dotnet/Program.cs index 4f51f9180..dbcfe1661 100644 --- a/src/dotnet/Program.cs +++ b/src/dotnet/Program.cs @@ -132,6 +132,8 @@ namespace Microsoft.DotNet.Cli command = "help"; } + telemetryClient.TrackEvent(command, null, null); + int exitCode; Func builtIn; if (s_builtIns.TryGetValue(command, out builtIn)) @@ -147,13 +149,6 @@ namespace Microsoft.DotNet.Cli exitCode = result.ExitCode; } - telemetryClient.TrackEvent( - command, - null, - new Dictionary - { - ["ExitCode"] = exitCode - }); return exitCode; diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 447322a51..4ae8e2644 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Threading.Tasks; using Microsoft.ApplicationInsights; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.InternalAbstractions; @@ -10,17 +11,27 @@ namespace Microsoft.DotNet.Cli public class Telemetry : ITelemetry { private bool _isInitialized = false; + private bool _isCollectingTelemetry = false; private TelemetryClient _client = null; private Dictionary _commonProperties = null; private Dictionary _commonMeasurements = null; + private Task _trackEventTask = null; + private int _sampleRate = 1; + private bool _isTestMachine = false; + + private const int ReciprocalSampleRateValue = 1; + private const int ReciprocalSampleRateValueForTest = 1; private const string InstrumentationKey = "74cc1c9e-3e6e-4d05-b3fc-dde9101d0254"; private const string TelemetryOptout = "DOTNET_CLI_TELEMETRY_OPTOUT"; + private const string TestMachineFlag = "TEST_MACHINE"; + private const string TestMachine = "Test Machine"; private const string OSVersion = "OS Version"; private const string OSPlatform = "OS Platform"; private const string RuntimeId = "Runtime Id"; private const string ProductVersion = "Product Version"; + private const string ReciprocalSampleRate = "Reciprocal SampleRate"; public bool Enabled { get; } @@ -33,26 +44,27 @@ namespace Microsoft.DotNet.Cli return; } + _sampleRate = ReciprocalSampleRateValue; + _isTestMachine = Env.GetEnvironmentVariableAsBool(TestMachineFlag); + + if(_isTestMachine) + { + _sampleRate = ReciprocalSampleRateValueForTest; + } + + _isCollectingTelemetry = (Environment.TickCount % _sampleRate == 0); + if(!_isCollectingTelemetry) + { + return; + } + try { using (PerfTrace.Current.CaptureTiming()) { - _client = new TelemetryClient(); - _client.InstrumentationKey = InstrumentationKey; - _client.Context.Session.Id = Guid.NewGuid().ToString(); - - - _client.Context.Device.OperatingSystem = RuntimeEnvironment.OperatingSystem; - - _commonProperties = new Dictionary(); - _commonProperties.Add(OSVersion, RuntimeEnvironment.OperatingSystemVersion); - _commonProperties.Add(OSPlatform, RuntimeEnvironment.OperatingSystemPlatform.ToString()); - _commonProperties.Add(RuntimeId, RuntimeEnvironment.GetRuntimeIdentifier()); - _commonProperties.Add(ProductVersion, Product.Version); - _commonMeasurements = new Dictionary(); + //initialize in task to offload to parallel thread + _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); } - - _isInitialized = true; } catch (Exception) { @@ -61,32 +73,83 @@ namespace Microsoft.DotNet.Cli Debug.Fail("Exception during telemetry initialization"); } } - + public void TrackEvent(string eventName, IDictionary properties, IDictionary measurements) { - if (!_isInitialized) + if (!Enabled) + { + return; + } + if (!_isCollectingTelemetry) + { + return; + } + try + { + using (PerfTrace.Current.CaptureTiming()) + { + _trackEventTask = _trackEventTask.ContinueWith( + x => TrackEventTask(eventName, properties, measurements) + ); + } + } + catch(Exception) + { + Debug.Fail("Exception during telemetry task continuation"); + } + } + + private void InitializeTelemetry() + { + try + { + _client = new TelemetryClient(); + _client.InstrumentationKey = InstrumentationKey; + _client.Context.Session.Id = Guid.NewGuid().ToString(); + + var runtimeEnvironment = PlatformServices.Default.Runtime; + _client.Context.Device.OperatingSystem = RuntimeEnvironment.OperatingSystem; + + _commonProperties = new Dictionary(); + _commonProperties.Add(OSVersion, RuntimeEnvironment.OperatingSystemVersion); + _commonProperties.Add(OSPlatform, RuntimeEnvironment.OperatingSystemPlatform.ToString()); + _commonProperties.Add(RuntimeId, RuntimeEnvironment.GetRuntimeIdentifier()); + _commonProperties.Add(ProductVersion, Product.Version); + _commonProperties.Add(TestMachine, _isTestMachine.ToString()); + _commonProperties.Add(ReciprocalSampleRate, _sampleRate.ToString()); + _commonMeasurements = new Dictionary(); + _isInitialized = true; + } + catch(Exception) + { + _isInitialized = false; + // we dont want to fail the tool if telemetry fails. + Debug.Fail("Exception during telemetry initialization"); + return; + } + } + + private void TrackEventTask(string eventName, IDictionary properties, IDictionary measurements) + { + if(!_isInitialized) { return; } - using (PerfTrace.Current.CaptureTiming()) + try { - Dictionary eventMeasurements = GetEventMeasures(measurements); - Dictionary eventProperties = GetEventProperties(properties); + var eventMeasurements = GetEventMeasures(measurements); + var eventProperties = GetEventProperties(properties); - try - { - _client.TrackEvent(eventName, eventProperties, eventMeasurements); - _client.Flush(); - } - catch (Exception) - { - Debug.Fail("Exception during TrackEvent"); - } + _client.TrackEvent(eventName, eventProperties, eventMeasurements); + _client.Flush(); + } + catch (Exception) + { + Debug.Fail("Exception during TrackEvent"); } } - private Dictionary GetEventMeasures(IDictionary measurements) { Dictionary eventMeasurements = new Dictionary(_commonMeasurements); From ba13586b1c01d67eba11b059a5086916e066f7fc Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Thu, 28 Apr 2016 14:49:17 -0700 Subject: [PATCH 2/9] Offloaded more work to Task Removed all unnecesary code if opted out of telemetry. Shanged sample rate to 1 for testing purposes. CI to just regular Test Changed hash helper function to handle a list to optimize unneceary duplicate sha256 creation Reduced new memory allocations --- src/dotnet/Telemetry.cs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 4ae8e2644..01e973740 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -58,6 +58,41 @@ namespace Microsoft.DotNet.Cli return; } + _isCollectingTelemetry = true; + try + { + //initialize in task to offload to parallel thread + _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); + } + catch(Exception) + { + Debug.Fail("Exception during telemetry task initialization"); + } + } + + public void TrackEvent(string eventName, IList properties, IDictionary measurements) + { + if (!_isCollectingTelemetry) + { + return; + } + + try + { + _trackEventTask = _trackEventTask.ContinueWith( + () => TrackEventTask(eventName, + properties, + measurements) + ); + } + catch(Exception) + { + Debug.Fail("Exception during telemetry task continuation"); + } + } + + private void InitializeTelemetry() + { try { using (PerfTrace.Current.CaptureTiming()) From 2d73e3b53511af162cd6a16c8987f893a0f51e04 Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Thu, 28 Apr 2016 16:32:56 -0700 Subject: [PATCH 3/9] Fixing inconsistencies that made build impossibe. --- src/dotnet/Telemetry.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 01e973740..1423aafe3 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -80,7 +80,7 @@ namespace Microsoft.DotNet.Cli try { _trackEventTask = _trackEventTask.ContinueWith( - () => TrackEventTask(eventName, + x => TrackEventTask(eventName, properties, measurements) ); From 702a44efc3a9e350bbafd3496b6037cbef5efba9 Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Mon, 2 May 2016 14:00:18 -0700 Subject: [PATCH 4/9] whitespace cleanup --- src/dotnet/Telemetry.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 1423aafe3..493aaacd5 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -52,7 +52,7 @@ namespace Microsoft.DotNet.Cli _sampleRate = ReciprocalSampleRateValueForTest; } - _isCollectingTelemetry = (Environment.TickCount % _sampleRate == 0); + _isCollectingTelemetry = (Environment.TickCount % _sampleRate == 0); if(!_isCollectingTelemetry) { return; @@ -69,19 +69,19 @@ namespace Microsoft.DotNet.Cli Debug.Fail("Exception during telemetry task initialization"); } } - + public void TrackEvent(string eventName, IList properties, IDictionary measurements) { if (!_isCollectingTelemetry) { return; } - + try { _trackEventTask = _trackEventTask.ContinueWith( - x => TrackEventTask(eventName, - properties, + x => TrackEventTask(eventName, + properties, measurements) ); } @@ -90,7 +90,7 @@ namespace Microsoft.DotNet.Cli Debug.Fail("Exception during telemetry task continuation"); } } - + private void InitializeTelemetry() { try @@ -108,7 +108,7 @@ namespace Microsoft.DotNet.Cli Debug.Fail("Exception during telemetry initialization"); } } - + public void TrackEvent(string eventName, IDictionary properties, IDictionary measurements) { if (!Enabled) @@ -118,7 +118,7 @@ namespace Microsoft.DotNet.Cli if (!_isCollectingTelemetry) { return; - } + } try { using (PerfTrace.Current.CaptureTiming()) @@ -223,9 +223,9 @@ namespace Microsoft.DotNet.Cli } return eventProperties; } - else + else { - return _commonProperties; + return _commonProperties; } } } From 42324ab9da6df1c07e00a4e3c2d9d84de681287c Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Mon, 2 May 2016 14:03:03 -0700 Subject: [PATCH 5/9] Removing accidental duplicate code --- src/dotnet/Telemetry.cs | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 493aaacd5..e10cc48eb 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -58,41 +58,6 @@ namespace Microsoft.DotNet.Cli return; } - _isCollectingTelemetry = true; - try - { - //initialize in task to offload to parallel thread - _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); - } - catch(Exception) - { - Debug.Fail("Exception during telemetry task initialization"); - } - } - - public void TrackEvent(string eventName, IList properties, IDictionary measurements) - { - if (!_isCollectingTelemetry) - { - return; - } - - try - { - _trackEventTask = _trackEventTask.ContinueWith( - x => TrackEventTask(eventName, - properties, - measurements) - ); - } - catch(Exception) - { - Debug.Fail("Exception during telemetry task continuation"); - } - } - - private void InitializeTelemetry() - { try { using (PerfTrace.Current.CaptureTiming()) @@ -101,11 +66,9 @@ namespace Microsoft.DotNet.Cli _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); } } - catch (Exception) + catch(Exception) { - // we dont want to fail the tool if telemetry fais. We should be able to detect abnormalities from data - // at the server end - Debug.Fail("Exception during telemetry initialization"); + Debug.Fail("Exception during telemetry task initialization"); } } From 5d548b339a281465efb386dc098af0183befad43 Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Mon, 2 May 2016 14:12:48 -0700 Subject: [PATCH 6/9] Misc cleanup --- src/dotnet/Telemetry.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index e10cc48eb..969bc1850 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -86,6 +86,7 @@ namespace Microsoft.DotNet.Cli { using (PerfTrace.Current.CaptureTiming()) { + //continue task in existing parallel thread _trackEventTask = _trackEventTask.ContinueWith( x => TrackEventTask(eventName, properties, measurements) ); @@ -105,7 +106,6 @@ namespace Microsoft.DotNet.Cli _client.InstrumentationKey = InstrumentationKey; _client.Context.Session.Id = Guid.NewGuid().ToString(); - var runtimeEnvironment = PlatformServices.Default.Runtime; _client.Context.Device.OperatingSystem = RuntimeEnvironment.OperatingSystem; _commonProperties = new Dictionary(); @@ -123,7 +123,6 @@ namespace Microsoft.DotNet.Cli _isInitialized = false; // we dont want to fail the tool if telemetry fails. Debug.Fail("Exception during telemetry initialization"); - return; } } @@ -136,15 +135,15 @@ namespace Microsoft.DotNet.Cli try { - var eventMeasurements = GetEventMeasures(measurements); var eventProperties = GetEventProperties(properties); + var eventMeasurements = GetEventMeasures(measurements); _client.TrackEvent(eventName, eventProperties, eventMeasurements); _client.Flush(); } catch (Exception) { - Debug.Fail("Exception during TrackEvent"); + Debug.Fail("Exception during TrackEventTask"); } } From 4aa22c33447a5208c1adff57dfff96db382ec3a2 Mon Sep 17 00:00:00 2001 From: Adam Gorman Date: Mon, 2 May 2016 15:31:33 -0700 Subject: [PATCH 7/9] Adding copyright header --- src/dotnet/Telemetry.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index 969bc1850..d5e31bb24 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -1,3 +1,6 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + using System; using System.Collections.Generic; using System.Diagnostics; From adbf30c1922b23febd6ee4265b8ddc96f0d277da Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Tue, 3 May 2016 15:09:27 -0500 Subject: [PATCH 8/9] Fixing GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms to give separate paths to the TestAssetsManager.CreateTestInstance by passing in a callingMethod name. --- ...otnetTestE2EInDesignTimeForMultipleTFms.cs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs index dd25f1dbe..765afe4b6 100644 --- a/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs +++ b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using FluentAssertions; using Microsoft.DotNet.InternalAbstractions; using Microsoft.DotNet.ProjectModel; @@ -14,13 +15,13 @@ namespace Microsoft.Dotnet.Tools.Test.Tests { public class GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms : TestBase { - private readonly string _projectFilePath; - private readonly string _netCoreAppOutputPath; - private readonly string _net451OutputPath; + private string _projectFilePath; + private string _netCoreAppOutputPath; + private string _net451OutputPath; - public GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms() + private void Setup([CallerMemberName] string callingMethod = "") { - var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject")); + var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject"), callingMethod); _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json"); var contexts = ProjectContext.CreateContextForEachFramework( @@ -51,6 +52,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [WindowsOnlyFact] public void It_discovers_tests_for_the_ProjectWithTestsWithNetCoreApp() { + Setup(); + using (var adapter = new Adapter("TestDiscovery.Start")) { adapter.Listen(); @@ -68,6 +71,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [WindowsOnlyFact] public void It_discovers_tests_for_the_ProjectWithTestsWithNet451() { + Setup(); + using (var adapter = new Adapter("TestDiscovery.Start")) { adapter.Listen(); @@ -86,6 +91,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [Fact] public void It_runs_tests_for_netcoreapp10() { + Setup(); + using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo")) { adapter.Listen(); @@ -105,6 +112,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [WindowsOnlyFact] public void It_runs_tests_for_net451() { + Setup(); + using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo")) { adapter.Listen(); From 6d698181a0c3373adb3e5e7a85ebdca4ca073768 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Tue, 3 May 2016 15:42:09 -0500 Subject: [PATCH 9/9] Add Telemetry Profile environment variable. Also removing unnecessary code in Telemetry.cs. --- src/dotnet/Telemetry.cs | 69 +++++++++-------------------------------- 1 file changed, 14 insertions(+), 55 deletions(-) diff --git a/src/dotnet/Telemetry.cs b/src/dotnet/Telemetry.cs index d5e31bb24..e9e0383fb 100644 --- a/src/dotnet/Telemetry.cs +++ b/src/dotnet/Telemetry.cs @@ -14,27 +14,22 @@ namespace Microsoft.DotNet.Cli public class Telemetry : ITelemetry { private bool _isInitialized = false; - private bool _isCollectingTelemetry = false; private TelemetryClient _client = null; private Dictionary _commonProperties = null; private Dictionary _commonMeasurements = null; private Task _trackEventTask = null; - private int _sampleRate = 1; - private bool _isTestMachine = false; + private string _telemetryProfile; - private const int ReciprocalSampleRateValue = 1; - private const int ReciprocalSampleRateValueForTest = 1; private const string InstrumentationKey = "74cc1c9e-3e6e-4d05-b3fc-dde9101d0254"; private const string TelemetryOptout = "DOTNET_CLI_TELEMETRY_OPTOUT"; - private const string TestMachineFlag = "TEST_MACHINE"; - private const string TestMachine = "Test Machine"; + private const string TelemetryProfileEnvironmentVariable = "DOTNET_CLI_TELEMETRY_PROFILE"; private const string OSVersion = "OS Version"; private const string OSPlatform = "OS Platform"; private const string RuntimeId = "Runtime Id"; private const string ProductVersion = "Product Version"; - private const string ReciprocalSampleRate = "Reciprocal SampleRate"; + private const string TelemetryProfile = "Telemetry Profile"; public bool Enabled { get; } @@ -47,32 +42,10 @@ namespace Microsoft.DotNet.Cli return; } - _sampleRate = ReciprocalSampleRateValue; - _isTestMachine = Env.GetEnvironmentVariableAsBool(TestMachineFlag); + _telemetryProfile = Environment.GetEnvironmentVariable(TelemetryProfileEnvironmentVariable); - if(_isTestMachine) - { - _sampleRate = ReciprocalSampleRateValueForTest; - } - - _isCollectingTelemetry = (Environment.TickCount % _sampleRate == 0); - if(!_isCollectingTelemetry) - { - return; - } - - try - { - using (PerfTrace.Current.CaptureTiming()) - { - //initialize in task to offload to parallel thread - _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); - } - } - catch(Exception) - { - Debug.Fail("Exception during telemetry task initialization"); - } + //initialize in task to offload to parallel thread + _trackEventTask = Task.Factory.StartNew(() => InitializeTelemetry()); } public void TrackEvent(string eventName, IDictionary properties, IDictionary measurements) @@ -81,24 +54,11 @@ namespace Microsoft.DotNet.Cli { return; } - if (!_isCollectingTelemetry) - { - return; - } - try - { - using (PerfTrace.Current.CaptureTiming()) - { - //continue task in existing parallel thread - _trackEventTask = _trackEventTask.ContinueWith( - x => TrackEventTask(eventName, properties, measurements) - ); - } - } - catch(Exception) - { - Debug.Fail("Exception during telemetry task continuation"); - } + + //continue task in existing parallel thread + _trackEventTask = _trackEventTask.ContinueWith( + x => TrackEventTask(eventName, properties, measurements) + ); } private void InitializeTelemetry() @@ -116,12 +76,11 @@ namespace Microsoft.DotNet.Cli _commonProperties.Add(OSPlatform, RuntimeEnvironment.OperatingSystemPlatform.ToString()); _commonProperties.Add(RuntimeId, RuntimeEnvironment.GetRuntimeIdentifier()); _commonProperties.Add(ProductVersion, Product.Version); - _commonProperties.Add(TestMachine, _isTestMachine.ToString()); - _commonProperties.Add(ReciprocalSampleRate, _sampleRate.ToString()); + _commonProperties.Add(TelemetryProfile, _telemetryProfile); _commonMeasurements = new Dictionary(); _isInitialized = true; } - catch(Exception) + catch (Exception) { _isInitialized = false; // we dont want to fail the tool if telemetry fails. @@ -131,7 +90,7 @@ namespace Microsoft.DotNet.Cli private void TrackEventTask(string eventName, IDictionary properties, IDictionary measurements) { - if(!_isInitialized) + if (!_isInitialized) { return; }