From 35751f7fdb3f137c52d1eddad712ceb9af7d3326 Mon Sep 17 00:00:00 2001 From: charlie <3140647@qq.com> Date: Fri, 29 Jul 2022 09:46:29 +0800 Subject: [PATCH] temporary --- cache/big_cache.go | 3 +- cache/cache.go | 45 ++++++++++++++----- logx/log_writer.go | 20 +++++++++ logx/logger.go | 1 + logx/logger_builder.go | 16 ------- logx/logger_builger_test.go | 6 --- ...ested_formatter.go => logrus_formatter.go} | 9 ++++ logx/logrus_options.go | 13 +++--- logx/logrus_test.go | 6 --- logx/muti_logger.go | 12 ----- logx/readme.md | 26 +++++++++++ logx/rotate_date_writer.go | 22 +++++++++ logx/rotate_size_writer.go | 3 ++ logx/rotate_writer_test.go | 27 +++++++++++ 14 files changed, 153 insertions(+), 56 deletions(-) create mode 100644 logx/logger.go delete mode 100644 logx/logger_builder.go rename logx/{logrus_nested_formatter.go => logrus_formatter.go} (76%) delete mode 100644 logx/muti_logger.go create mode 100644 logx/readme.md create mode 100644 logx/rotate_date_writer.go create mode 100644 logx/rotate_size_writer.go create mode 100644 logx/rotate_writer_test.go diff --git a/cache/big_cache.go b/cache/big_cache.go index 4fc7dd3..385ef2e 100644 --- a/cache/big_cache.go +++ b/cache/big_cache.go @@ -58,7 +58,8 @@ func (c *bigCacheClient) Set(key string, entry []byte, expire time.Duration) err } func (c *bigCacheClient) Delete(keys ...string) error { - for _, k := range keys { + ks := keys[:] + for _, k := range ks { if err := c.cache.Delete(k); err != nil { return err } diff --git a/cache/cache.go b/cache/cache.go index 5d26db2..ea02f1a 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -3,9 +3,11 @@ package cache import ( "context" "errors" + "fmt" "time" "github.com/charlienet/go-mixed/bytesconv" + "github.com/charlienet/go-mixed/locker" "github.com/charlienet/go-mixed/logx" ) @@ -14,13 +16,14 @@ var ErrNotFound = errors.New("key not found") type LoadFunc func(context.Context) (any, error) type Cache struct { - prefix string // 键前缀 - retry int // 资源获取时的重试次数 - mem MemCache // 内存缓存 - distributdCache DistributdCache // 分布式缓存 - publishSubscribe PublishSubscribe // 发布订阅 - qps *qps // - logger logx.Logger // 日志记录 + prefix string // 键前缀 + retry int // 资源获取时的重试次数 + mem MemCache // 内存缓存 + distributdCache DistributdCache // 分布式缓存 + publishSubscribe PublishSubscribe // 发布订阅 + lock locker.ChanLocker // 资源锁 + qps *qps // 访问计数 + logger logx.Logger // 日志记录 } func NewCache(opts ...option) *Cache { @@ -112,14 +115,36 @@ func (c *Cache) getFromMem(key string, out any) error { // 从缓存加载数据 func (c *Cache) getFromCache() { - + // 从缓存加载数据 + // 1. 检查内存是否存在 + // 2. 检查分布缓存是否存在 } // 从数据源加载数据 -func (c *Cache) getFromSource(ctx context.Context, key string, fn LoadFunc) { +func (c *Cache) getFromSource(ctx context.Context, key string, fn LoadFunc) error { // 1. 尝试获取资源锁，如成功获取到锁加载数据 // 2. 未获取到锁，等待从缓存中获取 - fn(ctx) + ch, ok := c.lock.Get(key) + if ok { + defer c.lock.Release(key) + v, err := fn(ctx) + if err != nil { + return fmt.Errorf("load from source err:%v", err) + } + + // 取出值存入多级缓存 + _ = v + + return nil + } + + // 等待数据加载完成 + select { + case <-ch: + + // 未取到结果时，再次获取 + return c.getFromSource(ctx, key, fn) + } } diff --git a/logx/log_writer.go b/logx/log_writer.go index e001260..6c1cdd5 100644 --- a/logx/log_writer.go +++ b/logx/log_writer.go @@ -1,3 +1,23 @@ package logx +import ( + "io" + "os" +) +type Rotate int + +const ( + None Rotate = iota // 不分割日志 + Size // 按大小分割 + Date // 按日期分割 +) + +type OutputOptions struct { + LogrusOutputOptions +} + +func WithFile(filename string) (io.Writer, error) { + mode := os.FileMode(0644) + return os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, mode) +} diff --git a/logx/logger.go b/logx/logger.go new file mode 100644 index 0000000..d083739 --- /dev/null +++ b/logx/logger.go @@ -0,0 +1 @@ +package logx diff --git a/logx/logger_builder.go b/logx/logger_builder.go deleted file mode 100644 index fafc8d6..0000000 --- a/logx/logger_builder.go +++ /dev/null @@ -1,16 +0,0 @@ -package logx - -type loggerBuilder struct { -} - -func NewBuilder() *loggerBuilder { - return &loggerBuilder{} -} - -func (b *loggerBuilder) WithLogrus() *loggerBuilder { - return b -} - -func (b *loggerBuilder) WithLogger() *loggerBuilder { - return b -} diff --git a/logx/logger_builger_test.go b/logx/logger_builger_test.go index d6c1f9d..8e4a203 100644 --- a/logx/logger_builger_test.go +++ b/logx/logger_builger_test.go @@ -2,14 +2,8 @@ package logx_test import ( "testing" - - "github.com/charlienet/go-mixed/logx" ) func TestBuilder(t *testing.T) { - logger := logx.NewBuilder(). - WithLogrus(). - WithLogger() - _ = logger } diff --git a/logx/logrus_nested_formatter.go b/logx/logrus_formatter.go similarity index 76% rename from logx/logrus_nested_formatter.go rename to logx/logrus_formatter.go index 8f992dc..419f17d 100644 --- a/logx/logrus_nested_formatter.go +++ b/logx/logrus_formatter.go @@ -6,6 +6,7 @@ import ( "runtime" nested "github.com/antonfisher/nested-logrus-formatter" + "github.com/sirupsen/logrus" ) const defaultTimestampFormat = "2006-01-02 15:04:05.000" @@ -14,6 +15,14 @@ type NestedFormatterOption struct { Color bool } +func NewJsonFormatter() logrus.Formatter { + return &logrus.JSONFormatter{} +} + +func NewTextFOrmatter() logrus.Formatter { + return &logrus.TextFormatter{} +} + func NewNestedFormatter(option NestedFormatterOption) *nested.Formatter { return &nested.Formatter{ TimestampFormat: defaultTimestampFormat, diff --git a/logx/logrus_options.go b/logx/logrus_options.go index fa06663..761ed23 100644 --- a/logx/logrus_options.go +++ b/logx/logrus_options.go @@ -4,6 +4,7 @@ import ( "io" "log" "os" + "time" "github.com/charlienet/go-mixed/fs" "github.com/sirupsen/logrus" @@ -32,11 +33,12 @@ type LogrusOutputOptions struct { } type LogrusBackupOptions struct { - MaxSize int // 默认大小100M - MaxAge int // 备份保留天数 - MaxBackups int // 备份保留数量 - LocalTime bool // 使用本地时间 - Compress bool // 是否压缩备份 + BackupType Rotate // 分割类型 + MaxSize int // 默认大小100M + MaxAge int // 备份保留天数 + MaxBackups int // 备份保留数量 + LocalTime bool // 使用本地时间 + Compress bool // 是否压缩备份 } func (o LogrusBackupOptions) hasBackup() bool { @@ -66,6 +68,7 @@ func WithFormatter(formatter logrus.Formatter) logrusOption { } func WithOutput(options LogrusOutputOptions) logrusOption { + _ = time.Now() return func(l *logrus.Logger) { var writer io.Writer switch { diff --git a/logx/logrus_test.go b/logx/logrus_test.go index bd4b4ab..1c1cc7a 100644 --- a/logx/logrus_test.go +++ b/logx/logrus_test.go @@ -39,9 +39,3 @@ func TestLevel(t *testing.T) { // logger.SetLevel(l) logger.Info("bcdefg") } - -func TestMutiWriter(t *testing.T) { - l := NewLogger().AppendLogger() - - _ = l -} diff --git a/logx/muti_logger.go b/logx/muti_logger.go deleted file mode 100644 index 1ab0ec4..0000000 --- a/logx/muti_logger.go +++ /dev/null @@ -1,12 +0,0 @@ -package logx - -type mutiLogger struct { -} - -func NewLogger() *mutiLogger { - return &mutiLogger{} -} - -func (w *mutiLogger) AppendLogger() Logger { - return nil -} diff --git a/logx/readme.md b/logx/readme.md new file mode 100644 index 0000000..a69d840 --- /dev/null +++ b/logx/readme.md @@ -0,0 +1,26 @@ +# 日志记录组件 + +日志分割及备份 + +日志可按照日期或大小进行分割，保留的历史日志文件数量由备份数量决定。 + +1. 按天拆分，每天生成新的日志文件名称。格式为file.yyyy-mm-dd.log， 其中file和log为配置的日志文件名称。 +2. 按大小拆分，使用lumberjack组件对日志文件进行分割。 +3. 按时间间隔拆分，日志文件按照指定的间隔拆分， + +日志输出流 +支持控制台和文件输出，可扩展输出组件 + +``` golang +logx.NewLogger( + WithLevel("debug"), + WithFormatter(), + WithConsole(), + WithRoateBySize(FileRoateSize{ + MaxSize + MaxAge + MaxBackups + }), + WithRoateByDate("filename", MaxAge, MaxBackups), + WithFile("filename")) +``` diff --git a/logx/rotate_date_writer.go b/logx/rotate_date_writer.go new file mode 100644 index 0000000..bf4bf94 --- /dev/null +++ b/logx/rotate_date_writer.go @@ -0,0 +1,22 @@ +package logx + +import ( + "io" +) + +// ensure we always implement io.WriteCloser +var _ io.WriteCloser = (*rotateDateWriter)(nil) + +type rotateDateWriter struct { + MaxAge int + MaxBackups int +} + +func (l *rotateDateWriter) Write(p []byte) (n int, err error) { + return 0, nil +} + +func (l *rotateDateWriter) Close() error { + + return nil +} diff --git a/logx/rotate_size_writer.go b/logx/rotate_size_writer.go new file mode 100644 index 0000000..07caf21 --- /dev/null +++ b/logx/rotate_size_writer.go @@ -0,0 +1,3 @@ +package logx + +// 按大小分割的日志记录器 diff --git a/logx/rotate_writer_test.go b/logx/rotate_writer_test.go new file mode 100644 index 0000000..7aa564b --- /dev/null +++ b/logx/rotate_writer_test.go @@ -0,0 +1,27 @@ +package logx + +import ( + "path/filepath" + "testing" + "time" + + rotatelogs "github.com/lestrrat/go-file-rotatelogs" +) + +func TestNewWriter(t *testing.T) { + t.Log(filepath.Abs("logs")) + + logf, err := rotatelogs.New("logs/aaaa.%Y%m%d.log", + rotatelogs.WithMaxAge(24*time.Hour), + rotatelogs.WithRotationTime(time.Hour)) + + if err != nil { + t.Fatal(err) + } + defer logf.Close() + + t.Log(logf.CurrentFileName()) + + _, err = logf.Write([]byte("abaccad")) + t.Log(err) +}