Log file rolling using vendored Serilog code (#2244)

This commit is contained in:
Mateusz Łach 2023-02-27 07:31:45 +01:00 committed by GitHub
parent 137a3a2882
commit 775f7b78d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 634 additions and 13 deletions

View File

@ -312,3 +312,8 @@ License Version 2.0
Library SharpCompress is under the following copyright:
Copyright (c) 2014 Adam Hathcock under the MIT License (MIT)
(<https://github.com/adamhathcock/sharpcompress/blob/master/LICENSE.txt>).
Component Serilog compiled into OpenTelemetry.AutoInstrumentation, OpenTelemetry.AutoInstrumentation.Loader, OpenTelemetry.AutoInstrumentation.StartupHook
is under the following copyright:
Copyright 2013-2017 Serilog Contributors under Apache License Version 2.0
(<https://github.com/serilog/serilog-sinks-file/blob/main/LICENSE>).

View File

@ -17,6 +17,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
using System.Text;
namespace OpenTelemetry.AutoInstrumentation.Loader

View File

@ -15,6 +15,21 @@
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\FileSink.cs">
<Link>Logging\FileSink.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\RollingFileSink.cs">
<Link>Logging\RollingFileSink.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\RollingInterval.cs">
<Link>Logging\RollingInterval.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\PathRoller.cs">
<Link>Logging\PathRoller.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\RollingLogFile.cs">
<Link>Logging\RollingLogFile.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\RollingIntervalExtensions.cs">
<Link>Logging\RollingIntervalExtensions.cs</Link>
</Compile>
<Compile Include="..\OpenTelemetry.AutoInstrumentation\Logging\IOtelLogger.cs">
<Link>Logging\IOtelLogger.cs</Link>
</Compile>

View File

@ -17,21 +17,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
using System.Text;
namespace OpenTelemetry.AutoInstrumentation.Logging;
internal sealed class FileSink : ISink, IDisposable
internal sealed class FileSink : IDisposable
{
readonly TextWriter _output;
readonly FileStream _underlyingStream;
readonly WriteCountingStream _countingStreamWrapper;
readonly object _syncRoot = new object();
static readonly long FileSizeLimitBytes = 10 * 1024 * 1024;
readonly long _fileSizeLimitBytes;
public FileSink(string path, Encoding encoding = null)
public FileSink(string path, long fileSizeLimitBytes)
{
if (path == null) throw new ArgumentNullException(nameof(path));
if (fileSizeLimitBytes < 1)
{
throw new ArgumentException("Invalid value provided; file size limit must be at least 1 byte.");
}
_fileSizeLimitBytes = fileSizeLimitBytes;
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(directory) && !Directory.Exists(directory))
@ -42,19 +49,20 @@ internal sealed class FileSink : ISink, IDisposable
_underlyingStream = System.IO.File.Open(path, FileMode.Append, FileAccess.Write, FileShare.Read);
Stream outputStream = _countingStreamWrapper = new WriteCountingStream(_underlyingStream);
_output = new StreamWriter(outputStream, encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
_output = new StreamWriter(outputStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
}
public void Write(string message)
public bool Write(string message)
{
lock (_syncRoot)
{
if (_countingStreamWrapper.CountedLength >= FileSizeLimitBytes)
if (_countingStreamWrapper.CountedLength >= _fileSizeLimitBytes)
{
return;
return false;
}
_output.Write(message);
FlushToDisk();
return true;
}
}
@ -66,7 +74,7 @@ internal sealed class FileSink : ISink, IDisposable
}
}
private void FlushToDisk()
public void FlushToDisk()
{
_output.Flush();
_underlyingStream.Flush(true);

View File

@ -29,6 +29,7 @@ internal static class OtelLogging
{
private const string OtelDotnetAutoLogDirectory = "OTEL_DOTNET_AUTO_LOG_DIRECTORY";
private const string NixDefaultDirectory = "/var/log/opentelemetry/dotnet";
private const int FileSizeLimitBytes = 10 * 1024 * 1024;
private static readonly ConcurrentDictionary<string, IOtelLogger> OtelLoggers = new();
@ -61,7 +62,13 @@ internal static class OtelLogging
{
var fileName = GetLogFileName(suffix);
var logPath = Path.Combine(logDirectory, fileName);
sink = new FileSink(logPath);
sink = new RollingFileSink(
path: logPath,
fileSizeLimitBytes: FileSizeLimitBytes,
retainedFileCountLimit: 10,
rollingInterval: RollingInterval.Day,
rollOnFileSizeLimit: true,
retainedFileTimeLimit: null);
}
}
catch (Exception)
@ -82,15 +89,15 @@ internal static class OtelLogging
var appDomainName = AppDomain.CurrentDomain.FriendlyName;
return string.IsNullOrEmpty(suffix)
? $"otel-dotnet-auto-{appDomainName}-{process.Id}.log"
: $"otel-dotnet-auto-{appDomainName}-{process.Id}-{suffix}.log";
? $"otel-dotnet-auto-{appDomainName}-{process.Id}-.log"
: $"otel-dotnet-auto-{appDomainName}-{process.Id}-{suffix}-.log";
}
catch
{
// We can't get the process info
return string.IsNullOrEmpty(suffix)
? $"otel-dotnet-auto-{Guid.NewGuid()}.log"
: $"otel-dotnet-auto-{Guid.NewGuid()}-{suffix}.log";
? $"otel-dotnet-auto-{Guid.NewGuid()}-.log"
: $"otel-dotnet-auto-{Guid.NewGuid()}-{suffix}-.log";
}
}

View File

@ -0,0 +1,135 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This comment is here to prevent StyleCop from analyzing a file originally from Serilog.
//------------------------------------------------------------------------------
// Copyright 2013-2016 Serilog Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
using System.Globalization;
using System.Text.RegularExpressions;
namespace OpenTelemetry.AutoInstrumentation.Logging;
internal class PathRoller
{
private const string PeriodMatchGroup = "period";
private const string SequenceNumberMatchGroup = "sequence";
private readonly Regex _filenameMatcher;
private readonly string _filenamePrefix;
private readonly string _filenameSuffix;
private readonly RollingInterval _interval;
private readonly string _periodFormat;
public PathRoller(string path, RollingInterval interval)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
_interval = interval;
_periodFormat = interval.GetFormat();
var pathDirectory = Path.GetDirectoryName(path);
if (string.IsNullOrEmpty(pathDirectory))
{
pathDirectory = Directory.GetCurrentDirectory();
}
LogFileDirectory = Path.GetFullPath(pathDirectory);
_filenamePrefix = Path.GetFileNameWithoutExtension(path);
_filenameSuffix = Path.GetExtension(path);
_filenameMatcher = new Regex(
"^" +
Regex.Escape(_filenamePrefix) +
"(?<" + PeriodMatchGroup + ">\\d{" + _periodFormat.Length + "})" +
"(?<" + SequenceNumberMatchGroup + ">_[0-9]{3,}){0,1}" +
Regex.Escape(_filenameSuffix) +
"$",
RegexOptions.Compiled);
DirectorySearchPattern = $"{_filenamePrefix}*{_filenameSuffix}";
}
public string LogFileDirectory { get; }
public string DirectorySearchPattern { get; }
public void GetLogFilePath(DateTime date, int? sequenceNumber, out string path)
{
var currentCheckpoint = GetCurrentCheckpoint(date);
var tok = currentCheckpoint?.ToString(_periodFormat, CultureInfo.InvariantCulture) ?? string.Empty;
if (sequenceNumber != null)
{
tok += "_" + sequenceNumber.Value.ToString("000", CultureInfo.InvariantCulture);
}
path = Path.Combine(LogFileDirectory, _filenamePrefix + tok + _filenameSuffix);
}
public IEnumerable<RollingLogFile> SelectMatches(IEnumerable<string> filenames)
{
foreach (var filename in filenames)
{
var match = _filenameMatcher.Match(filename);
if (!match.Success)
{
continue;
}
int? inc = null;
var incGroup = match.Groups[SequenceNumberMatchGroup];
if (incGroup.Captures.Count != 0)
{
var incPart = incGroup.Captures[0].Value.Substring(1);
inc = int.Parse(incPart, CultureInfo.InvariantCulture);
}
DateTime? period = null;
var periodGroup = match.Groups[PeriodMatchGroup];
if (periodGroup.Captures.Count != 0)
{
var dateTimePart = periodGroup.Captures[0].Value;
if (DateTime.TryParseExact(
dateTimePart,
_periodFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var dateTime))
{
period = dateTime;
}
}
yield return new RollingLogFile(filename, period, inc);
}
}
public DateTime? GetCurrentCheckpoint(DateTime instant)
{
return _interval.GetCurrentCheckpoint(instant);
}
public DateTime? GetNextCheckpoint(DateTime instant)
{
return _interval.GetNextCheckpoint(instant);
}
}

View File

@ -0,0 +1,256 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This comment is here to prevent StyleCop from analyzing a file originally from Serilog.
//------------------------------------------------------------------------------
// Copyright 2013-2017 Serilog Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
using System.Text;
namespace OpenTelemetry.AutoInstrumentation.Logging
{
sealed class RollingFileSink : ISink, IDisposable
{
readonly PathRoller _roller;
readonly long _fileSizeLimitBytes;
readonly int? _retainedFileCountLimit;
readonly TimeSpan? _retainedFileTimeLimit;
readonly bool _rollOnFileSizeLimit;
readonly object _syncRoot = new object();
bool _isDisposed;
DateTime? _nextCheckpoint;
#pragma warning disable CS8669
FileSink? _currentFile;
#pragma warning restore CS8669
int? _currentFileSequence;
public RollingFileSink(string path,
long fileSizeLimitBytes,
int? retainedFileCountLimit,
RollingInterval rollingInterval,
bool rollOnFileSizeLimit,
TimeSpan? retainedFileTimeLimit)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
if (fileSizeLimitBytes < 1)
{
throw new ArgumentException("Invalid value provided; file size limit must be at least 1 byte.");
}
if (retainedFileCountLimit.HasValue && retainedFileCountLimit < 1)
{
throw new ArgumentException("Zero or negative value provided; retained file count limit must be at least 1");
}
if (retainedFileTimeLimit.HasValue && retainedFileTimeLimit < TimeSpan.Zero)
{
throw new ArgumentException("Negative value provided; retained file time limit must be non-negative.", nameof(retainedFileTimeLimit));
}
_roller = new PathRoller(path, rollingInterval);
_fileSizeLimitBytes = fileSizeLimitBytes;
_retainedFileCountLimit = retainedFileCountLimit;
_retainedFileTimeLimit = retainedFileTimeLimit;
_rollOnFileSizeLimit = rollOnFileSizeLimit;
}
public void Write(string message)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
lock (_syncRoot)
{
if (_isDisposed)
{
throw new ObjectDisposedException("The log file has been disposed.");
}
var now = DateTime.Now;
AlignCurrentFileTo(now);
while (_currentFile?.Write(message) == false && _rollOnFileSizeLimit)
{
AlignCurrentFileTo(now, nextSequence: true);
}
}
}
void AlignCurrentFileTo(DateTime now, bool nextSequence = false)
{
if (!_nextCheckpoint.HasValue)
{
OpenFile(now);
}
else if (nextSequence || now >= _nextCheckpoint.Value)
{
int? minSequence = null;
if (nextSequence)
{
if (_currentFileSequence == null)
minSequence = 1;
else
minSequence = _currentFileSequence.Value + 1;
}
CloseFile();
OpenFile(now, minSequence);
}
}
void OpenFile(DateTime now, int? minSequence = null)
{
var currentCheckpoint = _roller.GetCurrentCheckpoint(now);
// We only try periodically because repeated failures
// to open log files REALLY slow an app down.
_nextCheckpoint = _roller.GetNextCheckpoint(now) ?? now.AddMinutes(30);
var existingFiles = Enumerable.Empty<string>();
try
{
if (Directory.Exists(_roller.LogFileDirectory))
{
existingFiles = Directory.GetFiles(_roller.LogFileDirectory, _roller.DirectorySearchPattern)
.Select(f => Path.GetFileName(f));
}
}
catch (DirectoryNotFoundException) { }
var latestForThisCheckpoint = _roller
.SelectMatches(existingFiles)
.Where(m => m.DateTime == currentCheckpoint)
.OrderByDescending(m => m.SequenceNumber)
.FirstOrDefault();
var sequence = latestForThisCheckpoint?.SequenceNumber;
if (minSequence != null)
{
if (sequence == null || sequence.Value < minSequence.Value)
sequence = minSequence;
}
const int maxAttempts = 3;
for (var attempt = 0; attempt < maxAttempts; attempt++)
{
_roller.GetLogFilePath(now, sequence, out var path);
try
{
_currentFile = new FileSink(path, _fileSizeLimitBytes);
_currentFileSequence = sequence;
}
catch (IOException)
{
sequence = (sequence ?? 0) + 1;
continue;
}
ApplyRetentionPolicy(path, now);
return;
}
}
void ApplyRetentionPolicy(string currentFilePath, DateTime now)
{
if (_retainedFileCountLimit == null && _retainedFileTimeLimit == null) return;
var currentFileName = Path.GetFileName(currentFilePath);
// We consider the current file to exist, even if nothing's been written yet,
// because files are only opened on response to an event being processed.
var potentialMatches = Directory.GetFiles(_roller.LogFileDirectory, _roller.DirectorySearchPattern)
.Select(f => Path.GetFileName(f))
.Union(new[] { currentFileName });
var newestFirst = _roller
.SelectMatches(potentialMatches)
.OrderByDescending(m => m.DateTime)
.ThenByDescending(m => m.SequenceNumber);
var toRemove = newestFirst
.Where(n => StringComparer.OrdinalIgnoreCase.Compare(currentFileName, n.Filename) != 0)
.SkipWhile((f, i) => ShouldRetainFile(f, i, now))
.Select(x => x.Filename)
.ToList();
foreach (var obsolete in toRemove)
{
var fullPath = Path.Combine(_roller.LogFileDirectory, obsolete);
try
{
System.IO.File.Delete(fullPath);
}
catch (Exception)
{
// unable to remove obsolete file
}
}
}
bool ShouldRetainFile(RollingLogFile file, int index, DateTime now)
{
if (_retainedFileCountLimit.HasValue && index >= _retainedFileCountLimit.Value - 1)
return false;
if (_retainedFileTimeLimit.HasValue && file.DateTime.HasValue &&
file.DateTime.Value < now.Subtract(_retainedFileTimeLimit.Value))
{
return false;
}
return true;
}
public void Dispose()
{
lock (_syncRoot)
{
if (_currentFile == null) return;
CloseFile();
_isDisposed = true;
}
}
void CloseFile()
{
if (_currentFile != null)
{
(_currentFile as IDisposable)?.Dispose();
_currentFile = null;
}
_nextCheckpoint = null;
}
public void FlushToDisk()
{
lock (_syncRoot)
{
_currentFile?.FlushToDisk();
}
}
}
}

View File

@ -0,0 +1,59 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This comment is here to prevent StyleCop from analyzing a file originally from Serilog.
//------------------------------------------------------------------------------
// Copyright 2017 Serilog Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
namespace OpenTelemetry.AutoInstrumentation.Logging
{
/// <summary>
/// Specifies the frequency at which the log file should roll.
/// </summary>
internal enum RollingInterval
{
/// <summary>
/// The log file will never roll; no time period information will be appended to the log file name.
/// </summary>
Infinite,
/// <summary>
/// Roll every year. Filenames will have a four-digit year appended in the pattern <code>yyyy</code>.
/// </summary>
Year,
/// <summary>
/// Roll every calendar month. Filenames will have <code>yyyyMM</code> appended.
/// </summary>
Month,
/// <summary>
/// Roll every day. Filenames will have <code>yyyyMMdd</code> appended.
/// </summary>
Day,
/// <summary>
/// Roll every hour. Filenames will have <code>yyyyMMddHH</code> appended.
/// </summary>
Hour,
/// <summary>
/// Roll every minute. Filenames will have <code>yyyyMMddHHmm</code> appended.
/// </summary>
Minute
}
}

View File

@ -0,0 +1,93 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This comment is here to prevent StyleCop from analyzing a file originally from Serilog.
//------------------------------------------------------------------------------
// Copyright 2017 Serilog Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
namespace OpenTelemetry.AutoInstrumentation.Logging
{
internal static class RollingIntervalExtensions
{
public static string GetFormat(this RollingInterval interval)
{
switch (interval)
{
case RollingInterval.Infinite:
return "";
case RollingInterval.Year:
return "yyyy";
case RollingInterval.Month:
return "yyyyMM";
case RollingInterval.Day:
return "yyyyMMdd";
case RollingInterval.Hour:
return "yyyyMMddHH";
case RollingInterval.Minute:
return "yyyyMMddHHmm";
default:
throw new ArgumentException("Invalid rolling interval");
}
}
public static DateTime? GetCurrentCheckpoint(this RollingInterval interval, DateTime instant)
{
switch (interval)
{
case RollingInterval.Infinite:
return null;
case RollingInterval.Year:
return new DateTime(instant.Year, 1, 1, 0, 0, 0, instant.Kind);
case RollingInterval.Month:
return new DateTime(instant.Year, instant.Month, 1, 0, 0, 0, instant.Kind);
case RollingInterval.Day:
return new DateTime(instant.Year, instant.Month, instant.Day, 0, 0, 0, instant.Kind);
case RollingInterval.Hour:
return new DateTime(instant.Year, instant.Month, instant.Day, instant.Hour, 0, 0, instant.Kind);
case RollingInterval.Minute:
return new DateTime(instant.Year, instant.Month, instant.Day, instant.Hour, instant.Minute, 0, instant.Kind);
default:
throw new ArgumentException("Invalid rolling interval");
}
}
public static DateTime? GetNextCheckpoint(this RollingInterval interval, DateTime instant)
{
var current = GetCurrentCheckpoint(interval, instant);
if (current == null)
{
return null;
}
switch (interval)
{
case RollingInterval.Year:
return current.Value.AddYears(1);
case RollingInterval.Month:
return current.Value.AddMonths(1);
case RollingInterval.Day:
return current.Value.AddDays(1);
case RollingInterval.Hour:
return current.Value.AddHours(1);
case RollingInterval.Minute:
return current.Value.AddMinutes(1);
default:
throw new ArgumentException("Invalid rolling interval");
}
}
}
}

View File

@ -0,0 +1,39 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This comment is here to prevent StyleCop from analyzing a file originally from Serilog.
//------------------------------------------------------------------------------
// Copyright 2013-2017 Serilog Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
namespace OpenTelemetry.AutoInstrumentation.Logging
{
internal class RollingLogFile
{
public RollingLogFile(string filename, DateTime? dateTime, int? sequenceNumber)
{
Filename = filename;
DateTime = dateTime;
SequenceNumber = sequenceNumber;
}
public string Filename { get; }
public DateTime? DateTime { get; }
public int? SequenceNumber { get; }
}
}

View File

@ -17,6 +17,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified by OpenTelemetry Authors.
namespace OpenTelemetry.AutoInstrumentation.Logging
{
sealed class WriteCountingStream : Stream