init

2025-11-04 21:09:16 +08:00
parent 8260e293c7
commit bb90a020dc
592 changed files with 61749 additions and 27 deletions
--- a/DouyinApi.Serilog/Configuration/LogBatchingSinkConfiguration.cs
+++ b/DouyinApi.Serilog/Configuration/LogBatchingSinkConfiguration.cs
@@ -0,0 +1,31 @@
+using DouyinApi.Common;
+using DouyinApi.Serilog.Sink;
+using Serilog;
+using Serilog.Sinks.PeriodicBatching;
+
+namespace DouyinApi.Serilog.Configuration;
+
+public static class LogBatchingSinkConfiguration
+{
+    public static LoggerConfiguration WriteToLogBatching(this LoggerConfiguration loggerConfiguration)
+    {
+        if (!AppSettings.app("AppSettings", "LogToDb").ObjToBool())
+        {
+            return loggerConfiguration;
+        }
+
+        var exampleSink = new LogBatchingSink();
+
+        var batchingOptions = new PeriodicBatchingSinkOptions
+        {
+            BatchSizeLimit = 500,
+            Period = TimeSpan.FromSeconds(1),
+            EagerlyEmitFirstEvent = true,
+            QueueLimit = 10000
+        };
+
+        var batchingSink = new PeriodicBatchingSink(exampleSink, batchingOptions);
+
+        return loggerConfiguration.WriteTo.Sink(batchingSink);
+    }
+}
--- a/DouyinApi.Serilog/DouyinApi.Serilog.csproj
+++ b/DouyinApi.Serilog/DouyinApi.Serilog.csproj
@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+
+	<ItemGroup>
+		<PackageReference Include="Mapster" Version="7.4.0" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\DouyinApi.Common\DouyinApi.Common.csproj" />
+	</ItemGroup>
+
+</Project>
--- a/DouyinApi.Serilog/Extensions/LoggerConfigurationExtensions.cs
+++ b/DouyinApi.Serilog/Extensions/LoggerConfigurationExtensions.cs
@@ -0,0 +1,83 @@
+using DouyinApi.Common;
+using DouyinApi.Common.LogHelper;
+using Serilog;
+using Serilog.Events;
+using Serilog.Filters;
+using SqlSugar;
+
+namespace DouyinApi.Serilog.Extensions;
+
+public static class LoggerConfigurationExtensions
+{
+    public static LoggerConfiguration WriteToConsole(this LoggerConfiguration loggerConfiguration)
+    {
+        //输出普通日志
+        loggerConfiguration = loggerConfiguration.WriteTo.Logger(lg =>
+            lg.FilterRemoveSqlLog().WriteTo.Console());
+
+        //输出SQL
+        loggerConfiguration = loggerConfiguration.WriteTo.Logger(lg =>
+            lg.FilterSqlLog().Filter.ByIncludingOnly(Matching.WithProperty<bool>(LogContextStatic.SqlOutToConsole, s => s))
+                .WriteTo.Console());
+
+        return loggerConfiguration;
+    }
+
+    public static LoggerConfiguration WriteToFile(this LoggerConfiguration loggerConfiguration)
+    {
+        //输出SQL
+        loggerConfiguration = loggerConfiguration.WriteTo.Logger(lg =>
+            lg.FilterSqlLog().Filter.ByIncludingOnly(Matching.WithProperty<bool>(LogContextStatic.SqlOutToFile, s => s))
+                .WriteTo.Async(s => s.File(LogContextStatic.Combine(LogContextStatic.AopSql, @"AopSql.txt"), rollingInterval: RollingInterval.Day,
+                    outputTemplate: LogContextStatic.FileMessageTemplate, retainedFileCountLimit: 31)));
+        //输出普通日志
+        loggerConfiguration = loggerConfiguration.WriteTo.Logger(lg =>
+            lg.FilterRemoveSqlLog().WriteTo.Async(s => s.File(LogContextStatic.Combine(LogContextStatic.BasePathLogs, @"Log.txt"), rollingInterval: RollingInterval.Day,
+                outputTemplate: LogContextStatic.FileMessageTemplate, retainedFileCountLimit: 31)));
+        return loggerConfiguration;
+    }
+
+    public static LoggerConfiguration FilterSqlLog(this LoggerConfiguration lc)
+    {
+        lc = lc.Filter.ByIncludingOnly(Matching.WithProperty<string>(LogContextStatic.LogSource, s => LogContextStatic.AopSql.Equals(s)));
+        return lc;
+    }
+
+    public static IEnumerable<LogEvent> FilterSqlLog(this IEnumerable<LogEvent> batch)
+    {
+        //只记录 Insert、Update、Delete语句
+        return batch.Where(s => s.WithProperty<string>(LogContextStatic.LogSource, q => LogContextStatic.AopSql.Equals(q)))
+            .Where(s => s.WithProperty<SugarActionType>(LogContextStatic.SugarActionType,
+                q => !new[] { SugarActionType.UnKnown, SugarActionType.Query }.Contains(q)));
+    }
+
+    public static LoggerConfiguration FilterRemoveSqlLog(this LoggerConfiguration lc)
+    {
+        lc = lc.Filter.ByIncludingOnly(WithProperty<string>(LogContextStatic.LogSource, s => !LogContextStatic.AopSql.Equals(s)));
+        return lc;
+    }
+
+    public static IEnumerable<LogEvent> FilterRemoveOtherLog(this IEnumerable<LogEvent> batch)
+    {
+        return batch.Where(s => WithProperty<string>(LogContextStatic.LogSource,
+            q => !LogContextStatic.AopSql.Equals(q))(s));
+    }
+
+    public static Func<LogEvent, bool> WithProperty<T>(string propertyName, Func<T, bool> predicate)
+    {
+        //如果不包含属性 也认为是true
+        return e =>
+        {
+            if (!e.Properties.TryGetValue(propertyName, out var propertyValue)) return true;
+
+            return propertyValue is ScalarValue { Value: T value } && predicate(value);
+        };
+    }
+
+    public static bool WithProperty<T>(this LogEvent e, string key, Func<T, bool> predicate)
+    {
+        if (!e.Properties.TryGetValue(key, out var propertyValue)) return false;
+
+        return propertyValue is ScalarValue { Value: T value } && predicate(value);
+    }
+}
--- a/DouyinApi.Serilog/Sink/LogBatchingSink.cs
+++ b/DouyinApi.Serilog/Sink/LogBatchingSink.cs
@@ -0,0 +1,135 @@
+using DouyinApi.Common;
+using DouyinApi.Model.Logs;
+using DouyinApi.Serilog.Extensions;
+using Mapster;
+using Serilog.Events;
+using Serilog.Sinks.PeriodicBatching;
+using SqlSugar;
+
+namespace DouyinApi.Serilog.Sink;
+
+public class LogBatchingSink : IBatchedLogEventSink
+{
+    public async Task EmitBatchAsync(IEnumerable<LogEvent> batch)
+    {
+        var sugar = App.GetService<ISqlSugarClient>(false);
+
+        await WriteSqlLog(sugar, batch.FilterSqlLog());
+        await WriteLogs(sugar, batch.FilterRemoveOtherLog());
+    }
+
+    public Task OnEmptyBatchAsync()
+    {
+        return Task.CompletedTask;
+    }
+
+    #region Write Log
+
+    private async Task WriteLogs(ISqlSugarClient db, IEnumerable<LogEvent> batch)
+    {
+        if (!batch.Any())
+        {
+            return;
+        }
+
+        var group = batch.GroupBy(s => s.Level);
+        foreach (var v in group)
+        {
+            switch (v.Key)
+            {
+                case LogEventLevel.Information:
+                    await WriteInformationLog(db, v);
+                    break;
+                case LogEventLevel.Warning:
+                    await WriteWarningLog(db, v);
+                    break;
+                case LogEventLevel.Error:
+                case LogEventLevel.Fatal:
+                    await WriteErrorLog(db, v);
+                    break;
+            }
+        }
+    }
+
+    private async Task WriteInformationLog(ISqlSugarClient db, IEnumerable<LogEvent> batch)
+    {
+        if (!batch.Any())
+        {
+            return;
+        }
+
+        var logs = new List<GlobalInformationLog>();
+        foreach (var logEvent in batch)
+        {
+            var log = logEvent.Adapt<GlobalInformationLog>();
+            log.Message = logEvent.RenderMessage();
+            log.Properties = logEvent.Properties.ToJson();
+            log.DateTime = logEvent.Timestamp.DateTime;
+            logs.Add(log);
+        }
+
+        await db.AsTenant().InsertableWithAttr(logs).SplitTable().ExecuteReturnSnowflakeIdAsync();
+    }
+
+    private async Task WriteWarningLog(ISqlSugarClient db, IEnumerable<LogEvent> batch)
+    {
+        if (!batch.Any())
+        {
+            return;
+        }
+
+        var logs = new List<GlobalWarningLog>();
+        foreach (var logEvent in batch)
+        {
+            var log = logEvent.Adapt<GlobalWarningLog>();
+            log.Message = logEvent.RenderMessage();
+            log.Properties = logEvent.Properties.ToJson();
+            log.DateTime = logEvent.Timestamp.DateTime;
+            logs.Add(log);
+        }
+
+        await db.AsTenant().InsertableWithAttr(logs).SplitTable().ExecuteReturnSnowflakeIdAsync();
+    }
+
+    private async Task WriteErrorLog(ISqlSugarClient db, IEnumerable<LogEvent> batch)
+    {
+        if (!batch.Any())
+        {
+            return;
+        }
+
+        var logs = new List<GlobalErrorLog>();
+        foreach (var logEvent in batch)
+        {
+            var log = logEvent.Adapt<GlobalErrorLog>();
+            log.Message = logEvent.RenderMessage();
+            log.Properties = logEvent.Properties.ToJson();
+            log.DateTime = logEvent.Timestamp.DateTime;
+            logs.Add(log);
+        }
+
+        await db.AsTenant().InsertableWithAttr(logs).SplitTable().ExecuteReturnSnowflakeIdAsync();
+    }
+
+    private async Task WriteSqlLog(ISqlSugarClient db, IEnumerable<LogEvent> batch)
+    {
+        if (!batch.Any())
+        {
+            return;
+        }
+
+        var logs = new List<AuditSqlLog>();
+        foreach (var logEvent in batch)
+        {
+            var log = logEvent.Adapt<AuditSqlLog>();
+            log.Message = logEvent.RenderMessage();
+            log.Properties = logEvent.Properties.ToJson();
+            log.DateTime = logEvent.Timestamp.DateTime;
+            logs.Add(log);
+        }
+
+        await db.AsTenant().InsertableWithAttr(logs).SplitTable().ExecuteReturnSnowflakeIdAsync();
+    }
+
+    #endregion
+}
--- a/DouyinApi.Serilog/Utility/SerilogRequestUtility.cs
+++ b/DouyinApi.Serilog/Utility/SerilogRequestUtility.cs
@@ -0,0 +1,73 @@
+using DouyinApi.Common.Extensions;
+using DouyinApi.Common.Https;
+using Microsoft.AspNetCore.Http;
+using Serilog;
+using Serilog.Events;
+
+namespace DouyinApi.Serilog.Utility;
+
+public class SerilogRequestUtility
+{
+	public const string HttpMessageTemplate =
+		"HTTP {RequestMethod} {RequestPath} QueryString:{QueryString} Body:{Body}  responded {StatusCode} in {Elapsed:0.0000} ms";
+
+	private static readonly List<string> _ignoreUrl = new()
+	{
+		"/job",
+	};
+
+	private static LogEventLevel DefaultGetLevel(HttpContext ctx,
+		double _,
+		Exception? ex)
+	{
+		return ex is null && ctx.Response.StatusCode <= 499 ? LogEventLevel.Information : LogEventLevel.Error;
+	}
+
+	public static LogEventLevel GetRequestLevel(HttpContext ctx, double _, Exception? ex) =>
+		ex is null && ctx.Response.StatusCode <= 499 ? IgnoreRequest(ctx) : LogEventLevel.Error;
+
+	private static LogEventLevel IgnoreRequest(HttpContext ctx)
+	{
+		var path = ctx.Request.Path.Value;
+		if (path.IsNullOrEmpty())
+		{
+			return LogEventLevel.Information;
+		}
+
+		return _ignoreUrl.Any(s => path.StartsWith(s)) ? LogEventLevel.Verbose : LogEventLevel.Information;
+	}
+
+	/// <summary>
+	/// 从Request中增加附属属性
+	/// </summary>
+	/// <param name="diagnosticContext"></param>
+	/// <param name="httpContext"></param>
+	public static void EnrichFromRequest(IDiagnosticContext diagnosticContext, HttpContext httpContext)
+	{
+		var request = httpContext.Request;
+
+		diagnosticContext.Set("RequestHost", request.Host);
+		diagnosticContext.Set("RequestScheme", request.Scheme);
+		diagnosticContext.Set("Protocol", request.Protocol);
+		diagnosticContext.Set("RequestIp", httpContext.GetRequestIp());
+
+		if (request.Method == HttpMethods.Get)
+		{
+			diagnosticContext.Set("QueryString", request.QueryString.HasValue ? request.QueryString.Value : string.Empty);
+			diagnosticContext.Set("Body", string.Empty);
+		}
+		else
+		{
+			diagnosticContext.Set("QueryString", request.QueryString.HasValue ? request.QueryString.Value : string.Empty);
+			diagnosticContext.Set("Body", request.ContentLength > 0 ? request.GetRequestBody() : string.Empty);
+		}
+
+		diagnosticContext.Set("ContentType", httpContext.Response.ContentType);
+
+		var endpoint = httpContext.GetEndpoint();
+		if (endpoint != null)
+		{
+			diagnosticContext.Set("EndpointName", endpoint.DisplayName);
+		}
+	}
+}