diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..1a8b21bc Binary files /dev/null and b/.DS_Store differ diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index 7bd11324..7d709715 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -21,8 +21,8 @@ using System.Security.AccessControl; using System.ServiceProcess; using System.Threading.Tasks; -using Seq.Forwarder.Cli.Features; -using Seq.Forwarder.Util; +using SeqCli.Forwarder.Cli.Features; +using SeqCli.Forwarder.Util; using SeqCli; using SeqCli.Cli; using SeqCli.Cli.Features; @@ -33,7 +33,7 @@ // ReSharper disable once ClassNeverInstantiated.Global -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "install", "Install the forwarder as a Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] @@ -95,7 +95,7 @@ int Setup() ServiceController controller; try { - Console.WriteLine("Checking the status of the Seq Forwarder service..."); + Console.WriteLine($"Checking the status of the {SeqCliForwarderWindowsService.WindowsServiceName} service..."); controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); Console.WriteLine("Status is {0}", controller.Status); diff --git a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs index 7d26d097..63008cb0 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RestartCommand.cs @@ -24,7 +24,7 @@ // ReSharper disable UnusedType.Global -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "restart", "Restart the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] diff --git a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs index 4c4cfa00..9a2a1b32 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/RunCommand.cs @@ -13,12 +13,11 @@ // limitations under the License. using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; -using System.Security.Cryptography.X509Certificates; +using System.Text; using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; @@ -30,13 +29,13 @@ using SeqCli.Config.Forwarder; using SeqCli.Forwarder; using SeqCli.Forwarder.Util; +using SeqCli.Forwarder.Web; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog; using Serilog.Core; using Serilog.Events; using Serilog.Formatting.Compact; -using Serilog.Formatting.Display; #if WINDOWS using SeqCli.Forwarder.ServiceProcess; @@ -79,6 +78,7 @@ protected override async Task Run(string[] unrecognized) try { + // ISSUE: we can't really rely on the default `SeqCliConfig` path being readable when running as a service. config = SeqCliConfig.Read(); // _storagePath.ConfigFilePath); } catch (Exception ex) @@ -107,7 +107,7 @@ protected override async Task Run(string[] unrecognized) { options.AddServerHeader = false; options.AllowSynchronousIO = true; - }).ConfigureKestrel((context, options) => + }).ConfigureKestrel((_, options) => { var apiListenUri = new Uri(listenUri); @@ -125,8 +125,8 @@ protected override async Task Run(string[] unrecognized) options.Listen(ipAddress, apiListenUri.Port, listenOptions => { #if WINDOWS - listenOptions.UseHttps(StoreName.My, apiListenUri.Host, - location: StoreLocation.LocalMachine, allowInvalid: true); + listenOptions.UseHttps(StoreName.My, apiListenUri.Host, + location: StoreLocation.LocalMachine, allowInvalid: true); #else listenOptions.UseHttps(); #endif @@ -138,31 +138,52 @@ protected override async Task Run(string[] unrecognized) } }); - builder - .Host.UseSerilog() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) - .ConfigureContainer(builder => + builder.Services.AddSerilog(); + + builder.Host.UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(containerBuilder => { - builder.RegisterBuildCallback(ls => container = ls); - builder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); + containerBuilder.RegisterBuildCallback(ls => container = ls); + containerBuilder.RegisterModule(new ForwarderModule(_storagePath.BufferPath, config)); }); - using var host = builder.Build(); + await using var app = builder.Build(); if (container == null) throw new Exception("Host did not build container."); + app.Use(async (context, next) => + { + try + { + await next(); + } + // ISSUE: this exception type isn't currently used. + catch (RequestProcessingException rex) + { + if (context.Response.HasStarted) + throw; + + context.Response.StatusCode = (int)rex.StatusCode; + context.Response.ContentType = "text/plain; charset=UTF-8"; + await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); + await context.Response.CompleteAsync(); + } + }); + foreach (var mapper in container.Resolve>()) { - mapper.Map(host); + mapper.MapEndpoints(app); } var service = container.Resolve( - new TypedParameter(typeof(IHost), host), + new TypedParameter(typeof(IHost), app), new NamedParameter("listenUri", listenUri)); var exit = ExecutionEnvironment.SupportsStandardIO - ? RunStandardIO(service, Console.Out) + ? await RunStandardIOAsync(service, Console.Out) : RunService(service); + + Log.Information("Exiting with status code {StatusCode}", exit); return exit; } @@ -178,6 +199,7 @@ protected override async Task Run(string[] unrecognized) } [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] + // ReSharper disable once UnusedParameter.Local static int RunService(ServerService service) { #if WINDOWS @@ -190,7 +212,7 @@ static int RunService(ServerService service) #endif } - static int RunStandardIO(ServerService service, TextWriter cout) + static async Task RunStandardIOAsync(ServerService service, TextWriter cout) { service.Start(); @@ -210,7 +232,7 @@ static int RunStandardIO(ServerService service, TextWriter cout) Console.Read(); } - service.Stop(); + await service.StopAsync(); return 0; } @@ -219,9 +241,9 @@ static void WriteBanner() { Write("─", ConsoleColor.DarkGray, 47); Console.WriteLine(); - Write(" Seq Forwarder", ConsoleColor.White); + Write(" SeqCli Forwarder", ConsoleColor.White); Write(" ──", ConsoleColor.DarkGray); - Write(" © 2024 Datalust Pty Ltd", ConsoleColor.Gray); + Write(" © Datalust Pty Ltd and Contributors", ConsoleColor.Gray); Console.WriteLine(); Write("─", ConsoleColor.DarkGray, 47); Console.WriteLine(); @@ -244,7 +266,7 @@ static Logger CreateLogger( var loggerConfiguration = new LoggerConfiguration() .Enrich.FromLogContext() .Enrich.WithProperty("MachineName", Environment.MachineName) - .Enrich.WithProperty("Application", "Seq Forwarder") + .Enrich.WithProperty("Application", "SeqCli Forwarder") .MinimumLevel.Is(internalLoggingLevel) .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) .WriteTo.File( diff --git a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs index dcbefb38..98ee92f8 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StartCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "start", "Start the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] diff --git a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs index 3eb6b0d3..3d0073b1 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StatusCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "status", "Show the status of the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] @@ -33,11 +33,11 @@ protected override Task Run() try { var controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - Console.WriteLine("The Seq Forwarder service is installed and {0}.", controller.Status.ToString().ToLowerInvariant()); + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is installed and {controller.Status.ToString().ToLowerInvariant()}."); } catch (InvalidOperationException) { - Console.WriteLine("The Seq Forwarder service is not installed."); + Console.WriteLine($"The {SeqCliForwarderWindowsService.WindowsServiceName} service is not installed."); } catch (Exception ex) { diff --git a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs index 59761a19..88d7db6b 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/StopCommand.cs @@ -22,7 +22,7 @@ using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "stop", "Stop the forwarder Windows service")] [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] diff --git a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs index 7ca95726..224cc94c 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/UninstallCommand.cs @@ -17,12 +17,12 @@ using System; using System.IO; using System.Threading.Tasks; -using Seq.Forwarder.Util; +using SeqCli.Forwarder.Util; using SeqCli.Cli; using SeqCli.Forwarder.ServiceProcess; using SeqCli.Forwarder.Util; -namespace Seq.Forwarder.Cli.Commands +namespace SeqCli.Forwarder.Cli.Commands { [Command("forwarder", "uninstall", "Uninstall the forwarder Windows service")] class UninstallCommand : Command diff --git a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs index 7e7fcd1a..60c2d250 100644 --- a/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs +++ b/src/SeqCli/Cli/Features/ServiceCredentialsFeature.cs @@ -16,7 +16,7 @@ using SeqCli.Cli; -namespace Seq.Forwarder.Cli.Features +namespace SeqCli.Forwarder.Cli.Features { class ServiceCredentialsFeature : CommandFeature { diff --git a/src/SeqCli/Cli/Features/StoragePathFeature.cs b/src/SeqCli/Cli/Features/StoragePathFeature.cs index 4283ddea..809a62b6 100644 --- a/src/SeqCli/Cli/Features/StoragePathFeature.cs +++ b/src/SeqCli/Cli/Features/StoragePathFeature.cs @@ -41,14 +41,14 @@ static string GetDefaultStorageRoot() // Specific to and writable by the current user. Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), #endif - "Seq", + "SeqCli", "Forwarder")); } static string? TryQueryInstalledStorageRoot() { #if WINDOWS - if (Seq.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( + if (SeqCli.Forwarder.Util.ServiceConfiguration.GetServiceStoragePath( SeqCliForwarderWindowsService.WindowsServiceName, out var storage)) return storage; #endif diff --git a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs index b5eea21f..6375dcf9 100644 --- a/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs +++ b/src/SeqCli/Forwarder/Diagnostics/InMemorySink.cs @@ -37,7 +37,7 @@ public IEnumerable Read() public void Emit(LogEvent logEvent) { - if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); + ArgumentNullException.ThrowIfNull(logEvent); _queue.Enqueue(logEvent); while (_queue.Count > _queueLength) diff --git a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs index 5e54a5bf..e3fbadf2 100644 --- a/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs +++ b/src/SeqCli/Forwarder/Diagnostics/IngestionLog.cs @@ -42,7 +42,7 @@ public static IEnumerable Read() return Sink.Read(); } - public static ILogger ForClient(IPAddress clientHostIP) + public static ILogger ForClient(IPAddress? clientHostIP) { return Log.ForContext("ClientHostIP", clientHostIP); } diff --git a/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs b/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs index b0885128..3dfbc691 100644 --- a/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs +++ b/src/SeqCli/Forwarder/ServiceProcess/SeqCliForwarderWindowsService.cs @@ -26,7 +26,7 @@ class SeqCliForwarderWindowsService : ServiceBase { readonly ServerService _serverService; - public static string WindowsServiceName { get; } = "Seq Forwarder"; + public static string WindowsServiceName { get; } = "SeqCli Forwarder"; public SeqCliForwarderWindowsService(ServerService serverService) { @@ -46,7 +46,7 @@ protected override void OnStart(string[] args) protected override void OnStop() { - _serverService.Stop(); + _serverService.StopAsync().Wait(); } } } diff --git a/src/SeqCli/Forwarder/Storage/LogBuffer.cs b/src/SeqCli/Forwarder/Storage/LogBuffer.cs index b53236cd..25bde0d9 100644 --- a/src/SeqCli/Forwarder/Storage/LogBuffer.cs +++ b/src/SeqCli/Forwarder/Storage/LogBuffer.cs @@ -5,7 +5,7 @@ namespace SeqCli.Forwarder.Storage; -record LogBuffer +class LogBuffer { public LogBuffer(Func write, CancellationToken cancellationToken) { @@ -42,7 +42,7 @@ public LogBuffer(Func write, CancellationToken cancella public async Task WriteAsync(byte[] storage, Range range, CancellationToken cancellationToken) { var tcs = new TaskCompletionSource(); - var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownTokenSource.Token); await _writer.WriteAsync(new LogBufferEntry(storage, range, tcs), cts.Token); await tcs.Task; diff --git a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs index b5b246fa..d9601814 100644 --- a/src/SeqCli/Forwarder/Storage/LogBufferMap.cs +++ b/src/SeqCli/Forwarder/Storage/LogBufferMap.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Serilog; namespace SeqCli.Forwarder.Storage; @@ -14,4 +15,10 @@ public LogBuffer Get(string? apiKey) { return new LogBuffer(async (c) => await Task.Delay(TimeSpan.FromSeconds(1), c), default); } + + public Task StopAsync() + { + Log.Information("Flushing log buffers"); + return Task.CompletedTask; + } } diff --git a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs index 6b73caf8..9074c532 100644 --- a/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs +++ b/src/SeqCli/Forwarder/Util/AccountRightsHelper.cs @@ -11,7 +11,7 @@ // ReSharper disable FieldCanBeMadeReadOnly.Local -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { public static class AccountRightsHelper { diff --git a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs b/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs deleted file mode 100644 index 6f78a435..00000000 --- a/src/SeqCli/Forwarder/Util/EnumerableExtensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace SeqCli.Forwarder.Util; - -static class EnumerableExtensions -{ - public static Dictionary ToDictionaryDistinct( - this IEnumerable enumerable, Func keySelector, Func valueSelector) - where TKey: notnull - { - var result = new Dictionary(); - foreach (var e in enumerable) - { - result[keySelector(e)] = valueSelector(e); - } - return result; - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs index 40b47071..4025c3eb 100644 --- a/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs +++ b/src/SeqCli/Forwarder/Util/ExecutionEnvironment.cs @@ -1,5 +1,5 @@ #if WINDOWS -using Seq.Forwarder.Util; +using SeqCli.Forwarder.Util; #endif namespace SeqCli.Forwarder.Util; diff --git a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs index f1739326..598a2e3c 100644 --- a/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs +++ b/src/SeqCli/Forwarder/Util/ServiceConfiguration.cs @@ -22,7 +22,7 @@ using System.Text; using SeqCli.Forwarder.Util; -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] public static class ServiceConfiguration diff --git a/src/SeqCli/Forwarder/Util/WindowsProcess.cs b/src/SeqCli/Forwarder/Util/WindowsProcess.cs index 98a20930..8e4d96a2 100644 --- a/src/SeqCli/Forwarder/Util/WindowsProcess.cs +++ b/src/SeqCli/Forwarder/Util/WindowsProcess.cs @@ -8,7 +8,7 @@ // ReSharper disable once InconsistentNaming -namespace Seq.Forwarder.Util +namespace SeqCli.Forwarder.Util { static class WindowsProcess { diff --git a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs index 90015f1b..822ecb23 100644 --- a/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/ApiRootEndpoints.cs @@ -12,41 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting.Display; namespace SeqCli.Forwarder.Web.Api; class ApiRootEndpoints : IMapEndpoints { - readonly MessageTemplateTextFormatter _formatter; - readonly Encoding Utf8 = new UTF8Encoding(false); + readonly Encoding _utf8 = new UTF8Encoding(false); - public ApiRootEndpoints(MessageTemplateTextFormatter formatter) + public void MapEndpoints(WebApplication app) { - _formatter = formatter; - } - - public void Map(WebApplication app) - { - app.MapGet("/", () => - { - var events = IngestionLog.Read(); - using var log = new StringWriter(); - foreach (var logEvent in events) - { - _formatter.Format(logEvent, log); - } - - return Results.Content(log.ToString(), "text/plain", Utf8); - }); - app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - + () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", _utf8)); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs index b5c53b1b..2a0812a5 100644 --- a/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IMapEndpoints.cs @@ -4,5 +4,5 @@ namespace SeqCli.Forwarder.Web.Api; interface IMapEndpoints { - void Map(WebApplication app); + void MapEndpoints(WebApplication app); } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 48fc8fa2..e511336e 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -53,11 +53,8 @@ public IngestionEndpoints( _logBuffers = logBuffers; } - public void Map(WebApplication app) + public void MapEndpoints(WebApplication app) { - app.MapGet("/api", - () => Results.Content("{\"Links\":{\"Events\":\"/api/events/describe\"}}", "application/json", Utf8)); - app.MapPost("api/events/raw", new Func>(async (context) => { var clef = DefaultedBoolQuery(context.Request, "clef"); @@ -66,67 +63,17 @@ public void Map(WebApplication app) return await IngestCompactFormat(context); var contentType = (string?) context.Request.Headers[HeaderNames.ContentType]; - var clefMediaType = "application/vnd.serilog.clef"; + const string clefMediaType = "application/vnd.serilog.clef"; if (contentType != null && contentType.StartsWith(clefMediaType)) return await IngestCompactFormat(context); - return IngestRawFormat(context); + IngestionLog.ForClient(context.Connection.RemoteIpAddress) + .Error("Client supplied a legacy raw-format (non-CLEF) payload"); + return Results.BadRequest("Only newline-delimited JSON (CLEF) payloads are supported."); })); } - IEnumerable EncodeRawEvents(ICollection events, IPAddress remoteIpAddress) - { - var encoded = new byte[events.Count][]; - var i = 0; - foreach (var e in events) - { - var s = e.ToString(Formatting.None); - var payload = Utf8.GetBytes(s); - - if (payload.Length > (int) _connectionConfig.EventBodyLimitBytes) - { - IngestionLog.ForPayload(remoteIpAddress, s).Debug("An oversized event was dropped"); - - var jo = e as JObject; - // ReSharper disable SuspiciousTypeConversion.Global - var timestamp = (string?) (dynamic?) jo?.GetValue("Timestamp") ?? DateTime.UtcNow.ToString("o"); - var level = (string?) (dynamic?) jo?.GetValue("Level") ?? "Warning"; - - if (jo != null) - { - jo.Remove("Timestamp"); - jo.Remove("Level"); - } - - var startToLog = (int) Math.Min(_connectionConfig.EventBodyLimitBytes / 2, 1024); - var compactPrefix = e.ToString(Formatting.None).Substring(0, startToLog); - - encoded[i] = Utf8.GetBytes(JsonConvert.SerializeObject(new - { - Timestamp = timestamp, - MessageTemplate = "Seq Forwarder received and dropped an oversized event", - Level = level, - Properties = new - { - Partial = compactPrefix, - Environment.MachineName, - _connectionConfig.EventBodyLimitBytes, - PayloadBytes = payload.Length - } - })); - } - else - { - encoded[i] = payload; - } - - i++; - } - - return encoded; - } - static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) { var parameter = request.Query[queryParameterName]; @@ -150,16 +97,7 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) var apiKeyHeader = request.Headers["X-SeqApiKey"]; if (apiKeyHeader.Count > 0) return apiKeyHeader.Last(); - if (request.Query.TryGetValue("apiKey", out var apiKey)) return apiKey.Last(); - - return null; - } - - - IResult IngestRawFormat(HttpContext context) - { - // Convert legacy format to CLEF - throw new NotImplementedException(); + return request.Query.TryGetValue("apiKey", out var apiKey) ? apiKey.Last() : null; } async Task IngestCompactFormat(HttpContext context) @@ -261,7 +199,7 @@ async Task IngestCompactFormat(HttpContext context) StatusCodes.Status201Created); } - bool ValidateClef(Span evt) + static bool ValidateClef(Span evt) { var reader = new Utf8JsonReader(evt); @@ -281,7 +219,7 @@ bool ValidateClef(Span evt) { var name = reader.GetString(); - if (name != null & name!.StartsWith("@")) + if (name != null & name!.StartsWith($"@")) { // Validate @ property } @@ -297,7 +235,7 @@ bool ValidateClef(Span evt) return true; } - async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) + static async Task Write(LogBuffer log, ArrayPool pool, byte[] storage, Range range, CancellationToken cancellationToken) { try { diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs new file mode 100644 index 00000000..2cbb3f8f --- /dev/null +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -0,0 +1,51 @@ +// Copyright Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using SeqCli.Forwarder.Diagnostics; +using Serilog.Formatting.Display; + +namespace SeqCli.Forwarder.Web.Api; + +class IngestionLogEndpoints : IMapEndpoints +{ + readonly MessageTemplateTextFormatter _formatter; + readonly Encoding _utf8 = new UTF8Encoding(false); + + public IngestionLogEndpoints(MessageTemplateTextFormatter formatter) + { + _formatter = formatter; + } + + public void MapEndpoints(WebApplication app) + { + // ISSUE: this route should probably only be mapped when some kind of --unsafe-debug flag + // is set. + + app.MapGet("/", () => + { + var events = IngestionLog.Read(); + using var log = new StringWriter(); + foreach (var logEvent in events) + { + _formatter.Format(logEvent, log); + } + + return Results.Content(log.ToString(), "text/plain", _utf8); + }); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 40934120..ee4b473b 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -13,8 +13,10 @@ // limitations under the License. using System; +using System.Threading.Tasks; using Microsoft.Extensions.Hosting; using SeqCli.Forwarder.Diagnostics; +using SeqCli.Forwarder.Storage; using Serilog; namespace SeqCli.Forwarder.Web.Host; @@ -22,11 +24,13 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; + readonly LogBufferMap _logBufferMap; readonly string _listenUri; - public ServerService(IHost host, string listenUri) + public ServerService(IHost host, LogBufferMap logBufferMap, string listenUri) { _host = host; + _logBufferMap = logBufferMap; _listenUri = listenUri; } @@ -38,8 +42,8 @@ public void Start() _host.Start(); - Log.Information("Seq Forwarder listening on {ListenUri}", _listenUri); - IngestionLog.Log.Debug("Seq Forwarder is accepting events"); + Log.Information("SeqCli Forwarder listening on {ListenUri}", _listenUri); + IngestionLog.Log.Debug("SeqCli Forwarder is accepting events"); } catch (Exception ex) { @@ -48,12 +52,14 @@ public void Start() } } - public void Stop() + public async Task StopAsync() { - Log.Debug("Seq Forwarder stopping"); + Log.Debug("Stopping HTTP server..."); - _host.StopAsync().Wait(); + await _host.StopAsync(); - Log.Information("Seq Forwarder stopped cleanly"); + Log.Information("HTTP server stopped; flushing buffers..."); + + await _logBufferMap.StopAsync(); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Host/Startup.cs b/src/SeqCli/Forwarder/Web/Host/Startup.cs deleted file mode 100644 index 3ca712a6..00000000 --- a/src/SeqCli/Forwarder/Web/Host/Startup.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Text; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; - -namespace SeqCli.Forwarder.Web.Host; - -class Startup -{ - public void ConfigureServices(IServiceCollection serviceCollection) - { - } - - public void Configure(IApplicationBuilder app) - { - app.Use(async (context, next) => - { - try - { - await next(); - } - catch (RequestProcessingException rex) - { - if (context.Response.HasStarted) - throw; - - context.Response.StatusCode = (int)rex.StatusCode; - context.Response.ContentType = "text/plain; charset=UTF-8"; - await context.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(rex.Message)); - await context.Response.CompleteAsync(); - } - }); - app.UseRouting(); - app.UseEndpoints(endpoints => - { - endpoints.MapControllers(); - }); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs deleted file mode 100644 index cee028fc..00000000 --- a/test/SeqCli.Tests/Forwarder/Storage/LogBufferTests.cs +++ /dev/null @@ -1,149 +0,0 @@ -using System.Collections.Generic; -using SeqCli.Forwarder.Storage; -using SeqCli.Tests.Support; -using Xunit; - -namespace SeqCli.Tests.Forwarder.Storage; - -public class LogBufferTests -{ - const ulong DefaultBufferSize = 10 * 1024 * 1024; - - [Fact] - public void ANewLogBufferIsEmpty() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Empty(contents); - } - - [Fact] - public void PeekingDoesNotChangeState() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - buffer.Enqueue([Some.Bytes(140)]); - - var contents = buffer.Peek((int)DefaultBufferSize); - Assert.Single(contents); - - var remainder = buffer.Peek((int)DefaultBufferSize); - Assert.Single(remainder); - } - - [Fact] - public void EnqueuedEntriesAreDequeuedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2]); - buffer.Enqueue([a3]); - - var contents = buffer.Peek((int)DefaultBufferSize); - - Assert.Equal(3, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - Assert.Equal(a3, contents[2].Value); - } - - [Fact] - public void EntriesOverLimitArePurgedFifo() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), 4096); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek((int)DefaultBufferSize); - - Assert.Equal(2, contents.Length); - Assert.Equal(a2, contents[0].Value); - Assert.Equal(a3, contents[1].Value); - } - - [Fact] - public void SizeHintLimitsDequeuedEventCount() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(300); - - Assert.Equal(2, contents.Length); - Assert.Equal(a1, contents[0].Value); - Assert.Equal(a2, contents[1].Value); - } - - [Fact] - public void AtLeastOneEventIsAlwaysDequeued() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(30); - - Assert.Single(contents); - Assert.Equal(a1, contents[0].Value); - } - - [Fact] - public void GivingTheLastSeenEventKeyRemovesPrecedingEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(420); - Assert.Equal(3, contents.Length); - - buffer.Dequeue(contents[2].Key); - - var remaining = buffer.Peek(420); - Assert.Empty(remaining); - } - - [Fact] - public void GivingTheLastSeeEventKeyLeavesSuccessiveEvents() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = buffer.Peek(30); - Assert.Single(contents); - - buffer.Enqueue([Some.Bytes(140)]); - - buffer.Dequeue(contents[0].Key); - - var remaining = buffer.Peek(420); - Assert.Equal(3, remaining.Length); - } - - [Fact] - public void EnumerationIsInOrder() - { - using var temp = TempFolder.ForCaller(); - using var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize); - byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140); - buffer.Enqueue([a1, a2, a3]); - - var contents = new List(); - buffer.Enumerate((k, v) => - { - contents.Add(v); - }); - - Assert.Equal(3, contents.Count); - Assert.Equal(new[] { a1, a2, a3 }, contents); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/SeqCli.Tests.csproj b/test/SeqCli.Tests/SeqCli.Tests.csproj index e93f0a96..9ebc2e63 100644 --- a/test/SeqCli.Tests/SeqCli.Tests.csproj +++ b/test/SeqCli.Tests/SeqCli.Tests.csproj @@ -15,7 +15,6 @@ - diff --git a/test/SeqCli.Tests/Support/TempFolder.cs b/test/SeqCli.Tests/Support/TempFolder.cs index 968fd857..8a5e85a0 100644 --- a/test/SeqCli.Tests/Support/TempFolder.cs +++ b/test/SeqCli.Tests/Support/TempFolder.cs @@ -15,7 +15,7 @@ public TempFolder(string name) { Path = System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), - "Seq.Forwarder.Tests", + "SeqCli.Forwarder.Tests", Session.ToString("n"), name);