From 1649bd5d5195a48c68788958e147b48f56060e7c Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Thu, 9 Nov 2023 10:33:56 +0800 Subject: [PATCH 1/6] Log trace logs to build output if not on TeamCity --- .../Support/LatestClientAndLatestServiceBuilder.cs | 2 +- source/Halibut.Tests/Support/SerilogLoggerBuilder.cs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/Halibut.Tests/Support/LatestClientAndLatestServiceBuilder.cs b/source/Halibut.Tests/Support/LatestClientAndLatestServiceBuilder.cs index 8a4b41319..87a675a1f 100644 --- a/source/Halibut.Tests/Support/LatestClientAndLatestServiceBuilder.cs +++ b/source/Halibut.Tests/Support/LatestClientAndLatestServiceBuilder.cs @@ -49,7 +49,7 @@ public class LatestClientAndLatestServiceBuilder : IClientAndServiceBuilder Reference? portForwarderReference; Func? pollingReconnectRetryPolicy; ProxyFactory? proxyFactory; - LogLevel halibutLogLevel = LogLevel.Trace; + LogLevel halibutLogLevel = LogLevel.Info; ConcurrentDictionary? clientInMemoryLoggers; ConcurrentDictionary? serviceInMemoryLoggers; ITrustProvider clientTrustProvider; diff --git a/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs b/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs index d447c312a..21a69055a 100644 --- a/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs +++ b/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs @@ -39,7 +39,7 @@ static SerilogLoggerBuilder() Logger = new LoggerConfiguration() .MinimumLevel.Verbose() - .WriteTo.Sink(new NonProgressNUnitSink(new MessageTemplateTextFormatter(nUnitOutputTemplate)), LogEventLevel.Debug) + .WriteTo.Sink(new NonProgressNUnitSink(new MessageTemplateTextFormatter(nUnitOutputTemplate))) .WriteTo.Sink(new TraceLogsForFailedTestsSink(new MessageTemplateTextFormatter(localOutputTemplate))) .CreateLogger(); } @@ -98,6 +98,12 @@ public void Emit(LogEvent logEvent) throw new ArgumentNullException(nameof(logEvent)); if (TestContext.Out == null) return; + + // SerilogLoggerBuilder creates this sink with Verbose logging, but we only want Verbose logging + // if we're running locally, as Verbose logs spam the TeamCity build log. + if (TeamCityDetection.IsRunningInTeamCity() && logEvent.Level < LogEventLevel.Debug) + return; + var output = new StringWriter(); if (logEvent.Properties.TryGetValue("SourceContext", out var sourceContext)) { From ad5c2665b43cb2dbea3dc440966cc98fb1966927 Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Fri, 10 Nov 2023 17:11:56 +0800 Subject: [PATCH 2/6] Everything in TraceLogFileLogger --- .../Support/SerilogLoggerBuilder.cs | 1 + source/Halibut.Tests/TraceLogFileLogger.cs | 31 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs b/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs index 21a69055a..bc8a0f0b7 100644 --- a/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs +++ b/source/Halibut.Tests/Support/SerilogLoggerBuilder.cs @@ -67,6 +67,7 @@ public ILogger Build() { TraceLoggers.AddOrUpdate(testName, traceFileLogger, (_, _) => throw new Exception("This should never be updated. If it is, it means that a test is being run multiple times in a single test run")); traceFileLogger.SetTestHash(testHash); + traceFileLogger.SetTestName(testName); } return logger; diff --git a/source/Halibut.Tests/TraceLogFileLogger.cs b/source/Halibut.Tests/TraceLogFileLogger.cs index cbb0ebf63..09799ae74 100644 --- a/source/Halibut.Tests/TraceLogFileLogger.cs +++ b/source/Halibut.Tests/TraceLogFileLogger.cs @@ -4,6 +4,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using JetBrains.TeamCity.ServiceMessages.Write; +using JetBrains.TeamCity.ServiceMessages.Write.Special; using Microsoft.VisualStudio.Threading; namespace Halibut.Tests @@ -13,6 +15,7 @@ public class TraceLogFileLogger : IDisposable readonly AsyncQueue queue = new(); readonly string tempFilePath = Path.GetTempFileName(); string testHash; + string testName; readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); readonly Task writeDataToDiskTask; @@ -27,6 +30,11 @@ public void SetTestHash(string testHash) this.testHash = testHash; } + public void SetTestName(string name) + { + this.testName = name; + } + public void WriteLine(string logMessage) { if (cancellationTokenSource.IsCancellationRequested) return; @@ -74,6 +82,12 @@ void FinishWritingLogs() public bool CopyLogFileToArtifacts() { FinishWritingLogs(); + return CopyFileToArtifactsDirectory(); + } + + bool CopyFileToArtifactsDirectory() + { + // The current directory is expected to have the following structure // (w/ variance depending on Debug/Release and dotnet framework used (net6.0, net48 etc): // @@ -87,9 +101,22 @@ public bool CopyLogFileToArtifacts() var traceLogsDirectory = rootDirectory.CreateSubdirectory("artifacts").CreateSubdirectory("trace-logs"); var fileName = $"{testHash}.tracelog"; + try { - File.Copy(tempFilePath, Path.Combine(traceLogsDirectory.ToString(), fileName), true); + var traceLogFilePath = Path.Combine(traceLogsDirectory.ToString(), fileName); + File.Copy(tempFilePath, traceLogFilePath, true); + + using var teamCityWriter = new TeamCityServiceMessages().CreateWriter(Console.WriteLine); + teamCityWriter.PublishArtifact($"{traceLogFilePath} => adrian-test-trace-logs"); + var artifactUri = $"adrian-test-trace-logs/{fileName}"; + teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") + { + { "testName", testName }, + { "type", "artifact" }, + { "value", artifactUri } + }); + return true; } catch @@ -111,4 +138,4 @@ public void Dispose() } } } -} \ No newline at end of file +} From f64c7f09158438168f1c19b8de5603931106830d Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Fri, 10 Nov 2023 17:26:08 +0800 Subject: [PATCH 3/6] Add failing test --- source/Halibut.Tests/UsageFixture.cs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source/Halibut.Tests/UsageFixture.cs b/source/Halibut.Tests/UsageFixture.cs index 026bd7555..9a5c1e391 100644 --- a/source/Halibut.Tests/UsageFixture.cs +++ b/source/Halibut.Tests/UsageFixture.cs @@ -16,6 +16,26 @@ namespace Halibut.Tests { public class UsageFixture : BaseTest { + [Test] + [LatestAndPreviousClientAndServiceVersionsTestCases()] + public async Task AlwaysFail(ClientAndServiceTestCase clientAndServiceTestCase) + { + await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() + .WithStandardServices() + .Build(CancellationToken)) + { + var echo = clientAndService.CreateAsyncClient(); + (await echo.SayHelloAsync("Deploy package A")).Should().Be("Deploy package A..."); + + for (var i = 0; i < clientAndServiceTestCase.RecommendedIterations; i++) + { + (await echo.SayHelloAsync($"Deploy package A {i}")).Should().Be($"Deploy package A {i}..."); + } + } + + Assert.Fail(); + } + [Test] [LatestAndPreviousClientAndServiceVersionsTestCases()] public async Task OctopusCanSendMessagesToTentacle_WithEchoService(ClientAndServiceTestCase clientAndServiceTestCase) From 09e69b1451cf8d13e4f55ff8126a76c5e8ab1e55 Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Tue, 14 Nov 2023 09:03:09 +0800 Subject: [PATCH 4/6] testwriter.writefile --- source/Halibut.Tests/TraceLogFileLogger.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/Halibut.Tests/TraceLogFileLogger.cs b/source/Halibut.Tests/TraceLogFileLogger.cs index 09799ae74..0cca4e7cb 100644 --- a/source/Halibut.Tests/TraceLogFileLogger.cs +++ b/source/Halibut.Tests/TraceLogFileLogger.cs @@ -110,12 +110,14 @@ bool CopyFileToArtifactsDirectory() using var teamCityWriter = new TeamCityServiceMessages().CreateWriter(Console.WriteLine); teamCityWriter.PublishArtifact($"{traceLogFilePath} => adrian-test-trace-logs"); var artifactUri = $"adrian-test-trace-logs/{fileName}"; - teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") - { - { "testName", testName }, - { "type", "artifact" }, - { "value", artifactUri } - }); + using var testWriter = teamCityWriter.OpenTest(testName); + testWriter.WriteFile(artifactUri, "Trace logs"); + // teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") + // { + // { "testName", testName }, + // { "type", "artifact" }, + // { "value", artifactUri } + // }); return true; } From 05499f267e16ad423baefc2309a320e3cea8214a Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Tue, 14 Nov 2023 09:23:42 +0800 Subject: [PATCH 5/6] write raw link msg --- source/Halibut.Tests/TraceLogFileLogger.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/Halibut.Tests/TraceLogFileLogger.cs b/source/Halibut.Tests/TraceLogFileLogger.cs index 0cca4e7cb..28d217473 100644 --- a/source/Halibut.Tests/TraceLogFileLogger.cs +++ b/source/Halibut.Tests/TraceLogFileLogger.cs @@ -111,13 +111,14 @@ bool CopyFileToArtifactsDirectory() teamCityWriter.PublishArtifact($"{traceLogFilePath} => adrian-test-trace-logs"); var artifactUri = $"adrian-test-trace-logs/{fileName}"; using var testWriter = teamCityWriter.OpenTest(testName); + testWriter.WriteValue("some random value", "Adrian test value"); testWriter.WriteFile(artifactUri, "Trace logs"); - // teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") - // { - // { "testName", testName }, - // { "type", "artifact" }, - // { "value", artifactUri } - // }); + teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") + { + { "testName", testName }, + { "type", "artifact" }, + { "value", artifactUri } + }); return true; } From cc054db1797261f9e6a60196acf5fa299b724b97 Mon Sep 17 00:00:00 2001 From: Adrian Codrington Date: Tue, 14 Nov 2023 16:07:47 +0800 Subject: [PATCH 6/6] Create test writer at start of test --- source/Halibut.Tests/TraceLogFileLogger.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/Halibut.Tests/TraceLogFileLogger.cs b/source/Halibut.Tests/TraceLogFileLogger.cs index 28d217473..a65ad526f 100644 --- a/source/Halibut.Tests/TraceLogFileLogger.cs +++ b/source/Halibut.Tests/TraceLogFileLogger.cs @@ -17,6 +17,9 @@ public class TraceLogFileLogger : IDisposable string testHash; string testName; + ITeamCityWriter teamCityWriter; + ITeamCityTestWriter testWriter; + readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); readonly Task writeDataToDiskTask; @@ -33,6 +36,8 @@ public void SetTestHash(string testHash) public void SetTestName(string name) { this.testName = name; + teamCityWriter = new TeamCityServiceMessages().CreateWriter(Console.WriteLine); + testWriter = teamCityWriter.OpenTest(testName); } public void WriteLine(string logMessage) @@ -106,11 +111,9 @@ bool CopyFileToArtifactsDirectory() { var traceLogFilePath = Path.Combine(traceLogsDirectory.ToString(), fileName); File.Copy(tempFilePath, traceLogFilePath, true); - - using var teamCityWriter = new TeamCityServiceMessages().CreateWriter(Console.WriteLine); + teamCityWriter.PublishArtifact($"{traceLogFilePath} => adrian-test-trace-logs"); var artifactUri = $"adrian-test-trace-logs/{fileName}"; - using var testWriter = teamCityWriter.OpenTest(testName); testWriter.WriteValue("some random value", "Adrian test value"); testWriter.WriteFile(artifactUri, "Trace logs"); teamCityWriter.WriteRawMessage(new ServiceMessage("testMetadata") @@ -132,6 +135,8 @@ public void Dispose() { try { + testWriter.Dispose(); + teamCityWriter.Dispose(); File.Delete(tempFilePath); } catch