diff --git a/cache/big_cache.go b/cache/big_cache.go index f2ba813..4fc7dd3 100644 --- a/cache/big_cache.go +++ b/cache/big_cache.go @@ -4,20 +4,42 @@ import ( "errors" "time" - "github.com/allegro/bigcache" + "github.com/allegro/bigcache/v3" + "github.com/charlienet/go-mixed/logx" ) var _ MemCache = &bigCacheClient{} type BigCacheConfig struct { + Shards int + LifeWindow time.Duration + CleanWindow time.Duration + MaxEntriesInWindow int + MaxEntrySize int + HardMaxCacheSize int + log logx.Logger } type bigCacheClient struct { cache *bigcache.BigCache } -func NewBigCache(c *BigCacheConfig) (*bigCacheClient, error) { - bigCache, err := bigcache.NewBigCache(bigcache.Config{}) +func NewBigCache(c BigCacheConfig) (*bigCacheClient, error) { + config := bigcache.DefaultConfig(time.Minute * 10) + + config.LifeWindow = c.LifeWindow + config.LifeWindow = c.LifeWindow + config.CleanWindow = c.CleanWindow + config.MaxEntriesInWindow = c.MaxEntriesInWindow + config.MaxEntrySize = c.MaxEntrySize + config.HardMaxCacheSize = c.HardMaxCacheSize + config.Logger = c.log + + if c.Shards > 0 { + config.Shards = c.Shards + } + + bigCache, err := bigcache.NewBigCache(config) if err != nil { return nil, err } @@ -35,8 +57,14 @@ func (c *bigCacheClient) Set(key string, entry []byte, expire time.Duration) err return c.cache.Set(key, entry) } -func (c *bigCacheClient) Delete(key string) error { - return c.cache.Delete(key) +func (c *bigCacheClient) Delete(keys ...string) error { + for _, k := range keys { + if err := c.cache.Delete(k); err != nil { + return err + } + } + + return nil } func (c *bigCacheClient) Exist(key string) { diff --git a/cache/cache.go b/cache/cache.go index 7e5c20e..5d26db2 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,36 +1,40 @@ package cache import ( + "context" "errors" "time" "github.com/charlienet/go-mixed/bytesconv" + "github.com/charlienet/go-mixed/logx" ) -var ErrNotFound = errors.New("not found") +var ErrNotFound = errors.New("key not found") -type LoadFunc func() (any, error) +type LoadFunc func(context.Context) (any, error) type Cache struct { prefix string // 键前缀 + retry int // 资源获取时的重试次数 mem MemCache // 内存缓存 distributdCache DistributdCache // 分布式缓存 publishSubscribe PublishSubscribe // 发布订阅 - qps *qps + qps *qps // + logger logx.Logger // 日志记录 } -func NewCache(opts ...option) (*Cache, error) { - c := &Cache{ - qps: NewQps(), - } +func NewCache(opts ...option) *Cache { + c := acquireDefaultCache() for _, f := range opts { - f(c) + if err := f(c); err != nil { + return c + } } go c.subscribe() - return c, nil + return c } func (c *Cache) Set(key string, value any, expiration time.Duration) error { @@ -60,13 +64,16 @@ func (c *Cache) Get(key string, out any) error { return nil } -func (c *Cache) GetFn(key string, out any, fn LoadFunc, expiration time.Duration) (bool, error) { - ret, err := fn() +func (c *Cache) GetFn(ctx context.Context, key string, out any, fn LoadFunc, expiration time.Duration) (bool, error) { + c.Get(key, out) + + // 多级缓存中未找到时,放置缓存对象 + ret, err := fn(ctx) if err != nil { return false, err } - _ = ret + c.Set(key, ret, expiration) return false, nil } @@ -75,20 +82,22 @@ func (c *Cache) Exist(key string) (bool, error) { return false, nil } -func (c *Cache) Delete(key string) error { +func (c *Cache) Delete(key ...string) error { if c.mem != nil { - c.mem.Delete(key) + c.mem.Delete(key...) } if c.distributdCache != nil { - c.distributdCache.Delete(key) + c.distributdCache.Delete(key...) } return nil } -func (c *Cache) getFromMem(key string, out any) error { +func (c *Cache) subscribe() { +} +func (c *Cache) getFromMem(key string, out any) error { bytes, err := c.mem.Get(key) if err != nil { return err @@ -101,13 +110,16 @@ func (c *Cache) getFromMem(key string, out any) error { return nil } -func (c *Cache) subscribe() { +// 从缓存加载数据 +func (c *Cache) getFromCache() { + } -func (c *Cache) genKey(key string) string { - if len(c.prefix) == 0 { - return key - } +// 从数据源加载数据 +func (c *Cache) getFromSource(ctx context.Context, key string, fn LoadFunc) { + + // 1. 尝试获取资源锁,如成功获取到锁加载数据 + // 2. 未获取到锁,等待从缓存中获取 + fn(ctx) - return c.prefix + "-" + key } diff --git a/cache/cache_builder.go b/cache/cache_builder.go new file mode 100644 index 0000000..4f18222 --- /dev/null +++ b/cache/cache_builder.go @@ -0,0 +1,99 @@ +package cache + +import "github.com/charlienet/go-mixed/logx" + +const defaultPrefix = "cache" + +type option func(*Cache) error + +type options struct { + Prefix string +} + +func acquireDefaultCache() *Cache { + return &Cache{ + prefix: defaultPrefix, + qps: NewQps(), + } +} + +type cacheBuilder struct { + prefix string + redisOptions RedisConfig + bigCacheConfig BigCacheConfig + freeSize int + publishSubscribe PublishSubscribe + log logx.Logger +} + +func NewCacheBuilder() *cacheBuilder { + return &cacheBuilder{} +} + +func (b *cacheBuilder) WithLogger(log logx.Logger) *cacheBuilder { + b.log = log + return b +} + +func (b *cacheBuilder) WithPrefix(prefix string) *cacheBuilder { + b.prefix = prefix + return b +} + +func (b *cacheBuilder) WithRedis(opts RedisConfig) *cacheBuilder { + b.redisOptions = opts + return b +} + +func (b *cacheBuilder) WithBigCache(opts BigCacheConfig) *cacheBuilder { + b.bigCacheConfig = opts + return b +} + +func (b *cacheBuilder) WithFreeCache(size int) *cacheBuilder { + b.freeSize = size + return b +} + +// 使用自定义分布式缓存 +func WithDistributedCache(c DistributdCache) { + +} + +func (b *cacheBuilder) WithPublishSubscribe(p PublishSubscribe) *cacheBuilder { + b.publishSubscribe = p + return b +} + +func (b cacheBuilder) Build() (*Cache, error) { + var err error + cache := acquireDefaultCache() + if len(b.prefix) > 0 { + cache.prefix = b.prefix + } + + b.redisOptions.Prefix = cache.prefix + + redis := NewRedis(b.redisOptions) + if err := redis.Ping(); err != nil { + return cache, err + } + + var mem MemCache + if b.freeSize > 0 { + mem = NewFreeCache(b.freeSize) + } else { + if b.log != nil { + b.bigCacheConfig.log = b.log + } + + mem, err = NewBigCache(b.bigCacheConfig) + } + + cache.distributdCache = redis + cache.mem = mem + cache.publishSubscribe = b.publishSubscribe + cache.logger = b.log + + return cache, err +} diff --git a/cache/cache_builder_test.go b/cache/cache_builder_test.go new file mode 100644 index 0000000..e3dca8d --- /dev/null +++ b/cache/cache_builder_test.go @@ -0,0 +1,29 @@ +package cache + +import ( + "testing" + "time" + + "github.com/charlienet/go-mixed/logx" +) + +func TestBuilder(t *testing.T) { + cache, err := NewCacheBuilder(). + WithLogger(logx.NewLogrus(logx.WithFormatter(logx.NewNestedFormatter(logx.NestedFormatterOption{ + Color: true, + })))). + WithRedis(RedisConfig{ + Addrs: []string{"192.168.2.222:6379"}, + Password: "123456", + }). + WithBigCache(BigCacheConfig{}). + // WithFreeCache(10 * 1024 * 1024). + Build() + + if err != nil { + t.Fatal(err) + } + + u := SimpleUser{FirstName: "Radomir", LastName: "Sohlich"} + t.Log(cache.Set(defaultKey, u, time.Minute*10)) +} diff --git a/cache/cache_test.go b/cache/cache_test.go index 91d0796..2c2e16c 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -1,24 +1,40 @@ package cache import ( + "context" + "sync" + "sync/atomic" "testing" "time" + + "github.com/charlienet/go-mixed/bytesconv" + "github.com/charlienet/go-mixed/logx" +) + +var ( + defaultKey = "u-000" ) func TestNewCache(t *testing.T) { - r := NewRedis(&RedisConfig{}) - if err := r.Ping(); err != nil { - t.Fatal(err) - } + c, err := NewCacheBuilder(). + WithRedis(RedisConfig{ + Addrs: []string{"192.168.2.222:6379"}, + Password: "123456", + }). + WithPrefix("cache_test"). + WithLogger(logx.NewLogrus()). + Build() - c, err := NewCache( - WithDistributdCache(r), - WithPrefix("cache_test")) if err != nil { t.Fatal(err) } c.Set("abc", "value", time.Minute*10) + + var s string + c.Get("abc", &s) + + t.Log(s) } type SimpleUser struct { @@ -26,60 +42,108 @@ type SimpleUser struct { LastName string } -func TestMem(t *testing.T) { - c, err := NewCache(WithFreeCache(10 * 1024 * 1024)) - if err != nil { - t.Fatal(err) +func TestMemCache(t *testing.T) { + b, _ := NewBigCache(BigCacheConfig{}) + var mems = []MemCache{ + NewFreeCache(10 * 1024 * 1024), + b, } - key := "u-000" + u := SimpleUser{FirstName: "Radomir", LastName: "Sohlich"} + encoded, _ := bytesconv.Encode(u) + for _, m := range mems { + m.Set(defaultKey, encoded, time.Second) + ret, err := m.Get(defaultKey) + if err != nil { + t.Fatal(err) + } - c.Set(key, u, time.Second) - - var u2 SimpleUser - c.Get(key, &u2) - - t.Logf("%+v", u2) + var u2 SimpleUser + bytesconv.Decode(ret, &u2) + t.Log(u2) + } } func TestDistributedCache(t *testing.T) { - key := "key-001" - c := NewRedis(&RedisConfig{Addrs: []string{"192.168.2.222:6379"}, DB: 6, Password: "123456"}) + c := NewRedis(RedisConfig{Addrs: []string{"192.168.2.222:6379"}, DB: 6, Password: "123456", Prefix: "abcdef"}) if err := c.Ping(); err != nil { t.Fatal(err) } + + t.Log(c.Exist(defaultKey)) + u := SimpleUser{FirstName: "redis client"} - c.Set(key, u, time.Second) var u2 SimpleUser - if err := c.Get(key, &u2); err != nil { + c.Get(defaultKey, &u2) + + c.Set(defaultKey, u, time.Minute*10) + t.Log(c.Exist(defaultKey)) + + if err := c.Get(defaultKey, &u2); err != nil { t.Fatal("err:", err) } t.Logf("%+v", u2) + + // c.Delete(defaultKey) } func TestGetFn(t *testing.T) { - c, err := NewCache(WithBigCache(&BigCacheConfig{})) - if err != nil { - t.Fatal(err) - } - key := "u-000" - + c := buildCache() var u2 SimpleUser - c.GetFn(key, &u2, func() (out any, err error) { + + c.GetFn(context.Background(), defaultKey, &u2, func(ctx context.Context) (out any, err error) { v := &u2 v.FirstName = "abc" + v.LastName = "aaaa" return nil, nil - }, time.Second) + }, time.Minute*1) t.Logf("%+v", u2) } +func TestGetFromSource(t *testing.T) { + var count int32 + + n := 10 + c := &Cache{} + wg := &sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func() { + c.getFromSource(context.Background(), defaultKey, func(ctx context.Context) (any, error) { + atomic.AddInt32(&count, 1) + time.Sleep(time.Second) + + return "abc", nil + }) + + wg.Done() + }() + } + + wg.Wait() + t.Log("count:", count) +} + func BenchmarkMemCache(b *testing.B) { } func load() (any, error) { return nil, nil } + +func buildCache() *Cache { + c, err := NewCacheBuilder(). + WithFreeCache(10 * 1024 * 1024). + WithRedis(RedisConfig{Addrs: []string{"192.168.2.222:6379"}, DB: 6, Password: "123456"}). + Build() + + if err != nil { + panic(err) + } + + return c +} diff --git a/cache/distributd_cache.go b/cache/distributd_cache.go index 5d7929a..2b05b8e 100644 --- a/cache/distributd_cache.go +++ b/cache/distributd_cache.go @@ -4,7 +4,7 @@ import "time" type DistributdCache interface { Get(key string, out any) error - Set(key string, value any, expiration time.Duration) - Delete(key string) error + Set(key string, value any, expiration time.Duration) error + Delete(key ...string) error Ping() error } diff --git a/cache/free_cache.go b/cache/free_cache.go index 4e9a4ae..5a0d64c 100644 --- a/cache/free_cache.go +++ b/cache/free_cache.go @@ -4,6 +4,7 @@ import ( "errors" "time" + "github.com/charlienet/go-mixed/bytesconv" "github.com/coocood/freecache" ) @@ -37,10 +38,13 @@ func (c *freeCache) Set(key string, value []byte, d time.Duration) error { return c.cache.Set([]byte(key), value, s) } -func (c *freeCache) Delete(key string) error { - affected := c.cache.Del([]byte(key)) - if !affected { - return errors.New("不存在") +func (c *freeCache) Delete(keys ...string) error { + for _, k := range keys { + affected := c.cache.Del(bytesconv.StringToBytes(k)) + + if !affected { + return errors.New("不存在") + } } return nil diff --git a/cache/mem.go b/cache/mem.go index 5f8cadf..d06f58d 100644 --- a/cache/mem.go +++ b/cache/mem.go @@ -5,5 +5,5 @@ import "time" type MemCache interface { Get(key string) ([]byte, error) Set(key string, entry []byte, expire time.Duration) error - Delete(key string) error + Delete(key ...string) error } diff --git a/cache/options.go b/cache/options.go deleted file mode 100644 index cf46d3e..0000000 --- a/cache/options.go +++ /dev/null @@ -1,31 +0,0 @@ -package cache - -type option func(*Cache) - -type options struct { - Prefix string -} - -func WithPrefix(prefix string) option { - return func(o *Cache) { o.prefix = prefix } -} - -func WithDistributdCache(d DistributdCache) option { - return func(o *Cache) { o.distributdCache = d } -} - -func WithBigCache(config *BigCacheConfig) option { - return func(o *Cache) { - c, err := NewBigCache(config) - _ = err - o.mem = c - } -} - -func WithFreeCache(size int) option { - return func(o *Cache) { o.mem = NewFreeCache(size) } -} - -func WithPublishSubscribe(p PublishSubscribe) option { - return func(o *Cache) {} -} diff --git a/cache/readme.md b/cache/readme.md new file mode 100644 index 0000000..056e5a8 --- /dev/null +++ b/cache/readme.md @@ -0,0 +1,13 @@ +# 多级缓存模块 + +1. 一级缓存可使用freecache或bigcache作为本地缓存,当数据在本地缓存不存在时,会向二级缓存请求数据 +2. 二级缓存使用redis作为缓存模块,当数据在二级缓存不存在时向资源请求数据。 +3. 更新数据时将二级分布式缓存中对应的数据删除,一级缓存使用订阅/发布机制进行删除。 + +## 功能列表 + +1. 支持自定义一级或二级缓存和订阅/发布机制。 +2. 缓存击穿;单机资源互斥锁,集群环境中每台机器只有一次请求到资源层,其它请求等待数据同步到缓存后获取数据。 +3. 缓存穿透;从数据源中未找到数据时,在缓存中缓存空值。 +4. 缓存雪崩;为防止缓存雪崩将资源放入缓存时,对过期时间添加一个随机过期时间,防止缓存同时过期。 +5. 自动续期;当访问二级缓存时对使用的资源进行延期。 diff --git a/cache/redis.go b/cache/redis.go index 1bac3e7..db9896b 100644 --- a/cache/redis.go +++ b/cache/redis.go @@ -2,6 +2,7 @@ package cache import ( "context" + "errors" "fmt" "time" @@ -11,8 +12,10 @@ import ( "github.com/go-redis/redis/v8" ) +const redisEmptyObject = "redis object not exist" + type RedisConfig struct { - Perfix string // key perfix + Prefix string // key perfix Addrs []string // Database to be selected after connecting to the server. @@ -33,10 +36,10 @@ type RedisConfig struct { type redisClient struct { client redis.UniversalClient emptyStamp string // 空对象标识,每个实例隔离 - perfix string // 缓存键前缀 + prefix string // 缓存键前缀 } -func NewRedis(c *RedisConfig) *redisClient { +func NewRedis(c RedisConfig) *redisClient { client := redis.NewUniversalClient(&redis.UniversalOptions{ Addrs: c.Addrs, DB: c.DB, @@ -46,35 +49,48 @@ func NewRedis(c *RedisConfig) *redisClient { return &redisClient{ emptyStamp: fmt.Sprintf("redis-empty-%d-%s", time.Now().Unix(), rand.Hex.Generate(6)), - perfix: c.Perfix, + prefix: c.Prefix, client: client, } } func (c *redisClient) Get(key string, out any) error { - cmd := c.client.Get(context.Background(), key) - str, err := cmd.Result() + val, err := c.client.Get(context.Background(), c.getKey(key)).Result() + if errors.Is(err, redis.Nil) { + return ErrNotFound + } + if err != nil { return err } - err = json.Unmarshal(bytesconv.StringToBytes(str), out) - return err + // redis 保存键为空值时返回键不存在错误 + if val == redisEmptyObject { + return ErrNotFound + } + + return json.Unmarshal(bytesconv.StringToBytes(val), out) } -func (c *redisClient) Set(key string, value any, expiration time.Duration) { +func (c *redisClient) Set(key string, value any, expiration time.Duration) error { j, _ := json.Marshal(value) - c.client.Set(context.Background(), key, j, expiration) + return c.client.Set(context.Background(), c.getKey(key), j, expiration).Err() } func (c *redisClient) Exist(key string) (bool, error) { - return false, nil + val, err := c.client.Exists(context.Background(), c.getKey(key)).Result() + return val > 0, err } -func (c *redisClient) Delete(key string) error { - cmd := c.client.Del(context.Background(), key) - if cmd.Err() != nil { - return cmd.Err() +func (c *redisClient) Delete(key ...string) error { + keys := make([]string, 0, len(key)) + for _, k := range key { + keys = append(keys, c.getKey(k)) + } + + _ , err := c.client.Del(context.Background(), keys...).Result() + if err != nil { + return err } return nil @@ -84,3 +100,11 @@ func (c *redisClient) Ping() error { _, err := c.client.Ping(context.Background()).Result() return err } + +func (c *redisClient) getKey(key string) string { + if c.prefix != "" { + return c.prefix + ":" + key + } + + return key +} diff --git a/crypto/rsa_test.go b/crypto/rsa_test.go index 6feeb69..34458e9 100644 --- a/crypto/rsa_test.go +++ b/crypto/rsa_test.go @@ -127,3 +127,41 @@ func TestBadPubKey(t *testing.T) { t.Log(rsa.Verify(msg, sign)) } + +func TestRsa2048Sign(t *testing.T) { + key := `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzLWIDZbQUyQh1+SLWxE6pS3lttbHSZo+B5NRB6R5nonuAmyf +XqQyJfDAvb2DTxXFL1y+VR2uEoQj/rg40PJ54ZnxQpEONEgSR7seIQr4xoe6h8SC +W0TcSBIejkzPkGPkWjsXczV74yQxz3+23SGSWiojEeWgjvQ6oI7SUC+yPb9UvaAx +GwMFCc0H5opSHW3ZVPdU6Q6bLK7QZXhY5CstXcxJ+bFQq3MPf+Dv5pB4qSR8PX02 +1pI/gmraEpbwU2Wqe+V07vA9ougJ/ON/xZtvuVT1xW3wazjC/QeBdWOjRhUsk611 +mo0retZ7nv72Tln5zBX8qPlJYNX5FQdv+LmRLwIDAQABAoIBAQC/F+6rkM9j7WTS +XnxgRJMUQuYfahua76tb8v8/PSBzCJrwFGopNOnDVRz3goOjPdVWwyLB3fTzP/tB ++sK++rsgCE6ZL0OtNmIqJ9iWS+GzolxUBPMTNBLWDGQNvlI8naM7P9JCL/k4Lj95 +TeVsQ7yVAqS+PjdFe2OHIgvd4shmrhttKYelXmjhoLi3SCIzU4Uj1RfBSzUM4Ffp +4FeuYCCxf2BSnweOa0DNREGlubMxcd+RQAChnDUbgdlpwjFJUlOmFYBCMn7U8ypG +PP3Z3/JI3glElVR7FQHS2yGvO4K9viHNr8eeDczKM+8VDs6ZT2ljpSXFei5Kv+Dv +tru4LlDxAoGBAOp8IarOlJnkDY4qoa0aN3ql0Z9yjCl7xy+7RytUjJvVLJbGkUpp +u6B+PJuVxa+OU4PKNSubU2gIcr+QaIo0tVyjZy2+zgSdddJLLc6q2wv7RqIsXdbk +PctfBAW+icu+ZSANyuI5bn220tAwQ9UO1F4bi/u/H64s6c5XN+i+9xn7AoGBAN99 +/WmOOnRJSNOTUMHjT5NNi/rh4UwSajB8S5nPsYUVB9Yj8CLbICzY3RWQkz6xbdoZ +e86IVDrcGa0cPWZyw/ecDCGt6JbfsYpzFmD1iQtrtBU0SfvZV47uAlN5d7CtAGKS +82rGCTNI+E65iKQY6D4WBygHFU2vX41OHFdq/pNdAoGBAIQf7diRDqqoFftFilQ/ +sYMqbDOsF85IMLR0kmWX/qLQO4+506Rab56/gucoPXvudqCMD+nCW/0CxaWreTxm +9sp8SGc+XFe9YeZc9jK9ky/tJp+64CV19lvh7iJOetaTMegd3XQbaGbt3Vvx1kb3 +VDKy0u3Hg9Jg/F2IR7id4h6BAoGACs2sUk3txXFFc/TLEpRKZHR7L8V7fpHlUDKx +9N11V1mM520VTpoJFCHnjgNPGti41rIkqfctGytIknWrAijKEE4ayAYAGEr36hlm +G4nC9ipeqie868+1y9L1idN1VbUHL7yqx56LE0+TsTqGwGfz0gx+jBDLltXDaLE9 +7XvekoECgYAc0QmX3E7NV/Ya41M1PAHA9/1caGa4nujKFtrKFUsTIhBuHn41F4aJ +JHIbDvh5OpqwlbdjMDT1JD8V0SqIKBgV0d//E7FySfuA1mW6AcXaf/AZavpG9r45 +rwxHC2OVdHfXo5MH7aI9Wax0CFdu2aZOEHLRFQE2KSXKY4/TcRdFpw== +-----END RSA PRIVATE KEY----- +` + v1 := []byte(`6217690700372158001X000000100TEST11573462713560839b241a0c54022019-11-11 17:01:211.00IACSPAY1911111701171012907901`) + + r, _ := crypto.NewRsa(crypto.SHA1, crypto.ParsePKCS1PrivateKey([]byte(key))) + ret, err := r.Sign(v1) + t.Log(err) + + t.Log("ret:", base64.StdEncoding.EncodeToString(ret)) +} diff --git a/go.mod b/go.mod index 6f53c25..a940fbb 100644 --- a/go.mod +++ b/go.mod @@ -17,22 +17,20 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( - github.com/allegro/bigcache v1.2.1 + github.com/allegro/bigcache/v3 v3.0.2 github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/coocood/freecache v1.2.1 - github.com/go-playground/assert/v2 v2.0.1 github.com/go-redis/redis/v8 v8.11.5 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.7.1 - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect - golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + github.com/stretchr/testify v1.7.2 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + golang.org/x/exp v0.0.0-20220608143224-64259d1afd70 + golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) diff --git a/go.sum b/go.sum index 47053d0..c69f30c 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache/v3 v3.0.2 h1:AKZCw+5eAaVyNTBmI2fgyPVJhHkdWder3O9IrprcQfI= +github.com/allegro/bigcache/v3 v3.0.2/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= @@ -23,8 +23,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -43,6 +41,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -67,18 +67,18 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd h1:zVFyTKZN/Q7mNRWSs1GOYnHM9NiFSJ54YVRsD0rNWT4= -golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20220608143224-64259d1afd70 h1:8uGxpY2cLF9H/NSHUiEWUIBZqIcsMzMWIMPCCUkyYgc= +golang.org/x/exp v0.0.0-20220608143224-64259d1afd70/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -93,15 +93,13 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0= +golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= @@ -131,8 +129,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/locker/rw_locker.go b/locker/rw_locker.go index 7370b64..73158df 100644 --- a/locker/rw_locker.go +++ b/locker/rw_locker.go @@ -2,8 +2,6 @@ package locker import "sync" -var _ RWLocker = &sync.RWMutex{} - func NewRWLocker() *sync.RWMutex { return &sync.RWMutex{} } diff --git a/locker/source_locker.go b/locker/source_locker.go index 82d214a..34bc0a3 100644 --- a/locker/source_locker.go +++ b/locker/source_locker.go @@ -1,21 +1,54 @@ package locker -import "sync" +import ( + "fmt" +) -var locks = make(map[string]sync.Locker) +// 资源锁 +type SourceLocker struct { + m RWLocker + locks map[string]Locker +} + +func NewSourceLocker() *SourceLocker { + return &SourceLocker{ + m: NewRWLocker(), + locks: make(map[string]Locker), + } +} + +func (s *SourceLocker) Lock(key string) { + s.m.RLock() + l, ok := s.locks[key] + + if ok { + s.m.RUnlock() -func Lock(name string) { - if l, ok := locks[name]; ok { l.Lock() - } + fmt.Println("加锁") + } else { + s.m.RUnlock() - new := &sync.Mutex{} - locks[name] = new - new.Lock() + s.m.Lock() + new := NewLocker() + s.locks[key] = new + s.m.Unlock() + + new.Lock() + fmt.Println("初始加锁") + } } -func Unlock(name string) { - if l, ok := locks[name]; ok { +func (s *SourceLocker) Unlock(key string) { + s.m.Lock() + if l, ok := s.locks[key]; ok { l.Unlock() + // delete(s.locks, key) + fmt.Println("解锁") } + s.m.Unlock() +} + +func (s *SourceLocker) TryLock(key string) bool { + return false } diff --git a/locker/source_locker_test.go b/locker/source_locker_test.go new file mode 100644 index 0000000..e0674c5 --- /dev/null +++ b/locker/source_locker_test.go @@ -0,0 +1,41 @@ +package locker + +import ( + "sync" + "testing" +) + +var sourcekey = "u-0001" + +func TestSourceLocker(t *testing.T) { + l := NewSourceLocker() + + c := 5 + n := 0 + wg := new(sync.WaitGroup) + wg.Add(c) + + for i := 0; i < c; i++ { + go func() { + defer wg.Done() + + l.Lock(sourcekey) + n++ + l.Unlock(sourcekey) + }() + } + + wg.Wait() + t.Log("n:", n) +} + +func BenchmarkSourceLocker(b *testing.B) { + l := NewSourceLocker() + + b.RunParallel(func(p *testing.PB) { + for p.Next() { + l.Lock(sourcekey) + l.Unlock(sourcekey) + } + }) +} diff --git a/locker/spin_locker_test.go b/locker/spin_locker_test.go new file mode 100644 index 0000000..229d615 --- /dev/null +++ b/locker/spin_locker_test.go @@ -0,0 +1,31 @@ +package locker + +import ( + "sync" + "testing" +) + +func TestSpinLock(t *testing.T) { + l := NewSpinLocker() + + n := 10 + c := 0 + + wg := new(sync.WaitGroup) + wg.Add(n) + for i := 0; i < n; i++ { + go func() { + defer wg.Done() + + l.Lock() + c++ + l.Unlock() + }() + } + + wg.Wait() + + l.Lock() + t.Log(c) + l.Unlock() +} diff --git a/maps/concurrence_test.go b/maps/concurrence_test.go index 25ad3ab..b0f2f78 100644 --- a/maps/concurrence_test.go +++ b/maps/concurrence_test.go @@ -2,7 +2,9 @@ package maps import ( "fmt" + "strconv" "sync" + "sync/atomic" "testing" "time" ) @@ -65,3 +67,57 @@ func BenchmarkMap(b *testing.B) { } }) } + +func BenchmarkLoadStore(b *testing.B) { + ms := []Map[string, string]{ + NewRWMap[string, string](), + NewConcurrentMap[string, string](), + NewHashMap[string, string]().Synchronize(), + NewHashMap[string, string](), + } + + for _, m := range ms { + var i int64 + b.Run(fmt.Sprintf("%T", m), func(b *testing.B) { + for n := 0; n < b.N; n++ { + gid := int(atomic.AddInt64(&i, 1) - 1) + + if gid == 0 { + m.Set("0", strconv.Itoa(n)) + } else { + m.Get("0") + } + } + }) + } +} + +func BenchmarkLoadStoreCollision(b *testing.B) { + ms := []Map[string, string]{ + NewRWMap[string, string](), + NewConcurrentMap[string, string](), + NewHashMap[string, string]().Synchronize(), + // &sync.Map{}, + } + + // 测试对于同一个 key 的 n-1 并发读和 1 并发写的性能 + for _, m := range ms { + b.Run(fmt.Sprintf("%T", m), func(b *testing.B) { + var i int64 + b.RunParallel(func(pb *testing.PB) { + // 记录并发执行的 goroutine id + gid := int(atomic.AddInt64(&i, 1) - 1) + + if gid == 0 { + for i := 0; pb.Next(); i++ { + m.Set("0", strconv.Itoa(i)) + } + } else { + for pb.Next() { + m.Get("0") + } + } + }) + }) + } +} diff --git a/pool/pool_test.go b/pool/pool_test.go index 12b8a63..e32ceac 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -33,6 +33,21 @@ func TestPoolSize(t *testing.T) { } } +func TestBytesPool(t *testing.T) { + var n = 0 + p := pool.NewPoolWithNew(100, func() []byte { + t.Log("new") + n++ + return make([]byte, 100, 100) + }) + + for i := 0; i < 1000; i++ { + go p.Put(p.Get()) + } + + t.Log("new count:", n) +} + func TestPut(t *testing.T) { p := pool.NewPool[PoolObject](10) for i := 0; i < 15; i++ { @@ -64,3 +79,7 @@ func BenchmarkPoolNew(b *testing.B) { } }) } + +func TestNewFunc(t *testing.T) { + +} diff --git a/rand/rand_test.go b/rand/rand_test.go index e82862b..0523658 100644 --- a/rand/rand_test.go +++ b/rand/rand_test.go @@ -3,6 +3,7 @@ package rand_test import ( "bytes" "fmt" + "math" "testing" "github.com/charlienet/go-mixed/rand" @@ -51,6 +52,7 @@ func TestRange(t *testing.T) { func TestFastrand(t *testing.T) { t.Log(int(^uint(0) >> 1)) + t.Log(math.MaxInt) for _, g := range generators { var max32 int32 = 1000 diff --git a/tests/string_test.go b/tests/string_test.go new file mode 100644 index 0000000..9e85aa0 --- /dev/null +++ b/tests/string_test.go @@ -0,0 +1,39 @@ +package tests + +import ( + "bytes" + "fmt" + "testing" +) + +func BenchmarkStringSplice(b *testing.B) { + userID := "aaaaa" + orderID := "bbccc" + + b.Run("BenchmarkPlus", func(b *testing.B) { + for i := 0; i < b.N; i++ { + logStr := "userid :" + userID + "; orderid:" + orderID + _ = logStr + } + }) + + b.Run("BenchmarkPrint", func(b *testing.B) { + for i := 0; i < b.N; i++ { + logStr := fmt.Sprintf("userid: %v; orderid: %v", userID, orderID) + _ = logStr + } + }) + + b.Run("BenchmarkBytesBuffer", func(b *testing.B) { + for i := 0; i < b.N; i++ { + var sb bytes.Buffer + sb.WriteString("userid :") + sb.WriteString(userID) + sb.WriteString("; orderid:") + sb.WriteString(orderID) + + logStr := sb.String() + _ = logStr + } + }) +}