1
0
mirror of https://github.com/charlienet/go-mixed.git synced 2025-07-18 08:32:40 +08:00

16 Commits

Author SHA1 Message Date
f043d2e5a7 布隆过滤器 2022-09-23 10:22:23 +08:00
5b4f8097d6 update 2022-09-01 12:37:57 +08:00
1071ad3694 RegisterFuzzyDecoders 2022-08-03 17:30:15 +08:00
35751f7fdb temporary 2022-07-29 09:46:29 +08:00
135b3a983b 添加掩码格式 2022-07-28 14:27:23 +08:00
52fabedd66 up 2022-07-26 14:20:09 +08:00
792458a185 update 2022-07-26 14:19:40 +08:00
23865214c8 mod 2022-07-26 14:18:20 +08:00
ebd76d2df6 abs 2022-07-26 14:15:32 +08:00
93352f03c1 cleanup guard 2022-07-26 14:15:13 +08:00
9c86470fa1 使用netip 2022-07-26 14:14:06 +08:00
37e9cabde8 snow flake 2022-07-25 14:51:16 +08:00
886723997e 优化range引用 2022-07-04 12:01:44 +08:00
44304f5b16 ip范围判断 2022-07-01 14:34:37 +08:00
f061b2efeb io.Writer 2022-06-27 16:05:37 +08:00
dcd803b4f2 hmac name 2022-06-27 14:41:21 +08:00
49 changed files with 1135 additions and 149 deletions

View File

@ -1,6 +1,9 @@
package bloom
import "github.com/bits-and-blooms/bitset"
import (
"github.com/bits-and-blooms/bitset"
"github.com/charlienet/go-mixed/locker"
)
const DEFAULT_SIZE = 2 << 24
@ -12,23 +15,53 @@ type simplehash struct {
}
type BloomFilter struct {
set *bitset.BitSet
funcs [6]simplehash
size int // 布隆过滤器大小
set *bitset.BitSet // 位图
funcs [6]simplehash // 哈希函数
lock locker.RWLocker
}
func NewBloomFilter() *BloomFilter {
bf := new(BloomFilter)
for i := 0; i < len(bf.funcs); i++ {
bf.funcs[i] = simplehash{DEFAULT_SIZE, seeds[i]}
type bloomOptions struct {
Size int
}
bf.set = bitset.New(DEFAULT_SIZE)
type option func(*bloomOptions)
// 布隆过滤器中所有位长度,请根据存储数量进行评估
func WithSize(size int) option {
return func(bo *bloomOptions) {
bo.Size = size
}
}
func NewBloomFilter(opts ...option) *BloomFilter {
opt := &bloomOptions{
Size: DEFAULT_SIZE,
}
for _, f := range opts {
f(opt)
}
bf := &BloomFilter{
size: opt.Size,
lock: locker.NewRWLocker(),
}
for i := 0; i < len(bf.funcs); i++ {
bf.funcs[i] = simplehash{uint(opt.Size), seeds[i]}
}
bf.set = bitset.New(uint(opt.Size))
return bf
}
func (bf *BloomFilter) Add(value string) {
for _, f := range bf.funcs {
funcs := bf.funcs[:]
for _, f := range funcs {
bf.set.Set(f.hash(value))
}
}
func (bf *BloomFilter) Contains(value string) bool {
@ -36,12 +69,20 @@ func (bf *BloomFilter) Contains(value string) bool {
return false
}
ret := true
for _, f := range bf.funcs {
funcs := bf.funcs[:]
for _, f := range funcs {
ret = ret && bf.set.Test(f.hash(value))
}
return ret
}
// 清空布隆过滤器
func (bf *BloomFilter) Clear() {
bf.set.ClearAll()
}
func (s simplehash) hash(value string) uint {
var result uint = 0
for i := 0; i < len(value); i++ {

View File

@ -6,6 +6,9 @@ import (
"testing"
"github.com/charlienet/go-mixed/bloom"
"github.com/charlienet/go-mixed/rand"
"github.com/charlienet/go-mixed/sys"
"github.com/stretchr/testify/assert"
)
func TestBloom(t *testing.T) {
@ -15,6 +18,58 @@ func TestBloom(t *testing.T) {
b.Add(strconv.Itoa(i))
}
fmt.Println(b.Contains(strconv.Itoa(9999)))
fmt.Println(b.Contains("ss"))
v := "6943553521463296-1635402930"
t.Log(b.Contains(v))
b.Add(v)
t.Log(b.Contains(v))
fmt.Println("过滤器中包含值:", b.Contains(strconv.Itoa(9999)))
fmt.Println("过滤器中未包含:", b.Contains("ss"))
t.Log(sys.ShowMemUsage())
}
func TestSize(t *testing.T) {
bloom.NewBloomFilter(bloom.WithSize(1 << 2))
}
func TestClear(t *testing.T) {
bf := bloom.NewBloomFilter()
v := "abc"
bf.Add(v)
assert.True(t, bf.Contains(v))
bf.Clear()
assert.False(t, bf.Contains(v))
}
func TestParallel(t *testing.T) {
f := bloom.NewBloomFilter()
for i := 0; i < 10000; i++ {
v := rand.Hex.Generate(10)
f.Add(v)
assert.True(t, f.Contains(v))
}
}
func BenchmarkFilter(b *testing.B) {
f := bloom.NewBloomFilter()
b.RunParallel(func(p *testing.PB) {
for p.Next() {
v := rand.Hex.Generate(10)
f.Add(v)
f.Contains(v)
// assert.True(b, f.Contains(v))
// assert.True(b, f.Contains(v))
}
})
}

View File

@ -16,7 +16,9 @@ func (r BytesResult) Hex() string {
func (r BytesResult) UppercaseHex() string {
dst := make([]byte, hex.EncodedLen(len(r)))
j := 0
for _, v := range r {
re := r[:]
for _, v := range re {
dst[j] = hextable[v>>4]
dst[j+1] = hextable[v&0x0f]
j += 2

3
cache/big_cache.go vendored
View File

@ -58,7 +58,8 @@ func (c *bigCacheClient) Set(key string, entry []byte, expire time.Duration) err
}
func (c *bigCacheClient) Delete(keys ...string) error {
for _, k := range keys {
ks := keys[:]
for _, k := range ks {
if err := c.cache.Delete(k); err != nil {
return err
}

33
cache/cache.go vendored
View File

@ -3,9 +3,11 @@ package cache
import (
"context"
"errors"
"fmt"
"time"
"github.com/charlienet/go-mixed/bytesconv"
"github.com/charlienet/go-mixed/locker"
"github.com/charlienet/go-mixed/logx"
)
@ -19,7 +21,8 @@ type Cache struct {
mem MemCache // 内存缓存
distributdCache DistributdCache // 分布式缓存
publishSubscribe PublishSubscribe // 发布订阅
qps *qps //
lock locker.ChanLocker // 资源锁
qps *qps // 访问计数
logger logx.Logger // 日志记录
}
@ -112,14 +115,36 @@ func (c *Cache) getFromMem(key string, out any) error {
// 从缓存加载数据
func (c *Cache) getFromCache() {
// 从缓存加载数据
// 1. 检查内存是否存在
// 2. 检查分布缓存是否存在
}
// 从数据源加载数据
func (c *Cache) getFromSource(ctx context.Context, key string, fn LoadFunc) {
func (c *Cache) getFromSource(ctx context.Context, key string, fn LoadFunc) error {
// 1. 尝试获取资源锁,如成功获取到锁加载数据
// 2. 未获取到锁,等待从缓存中获取
fn(ctx)
ch, ok := c.lock.Get(key)
if ok {
defer c.lock.Release(key)
v, err := fn(ctx)
if err != nil {
return fmt.Errorf("load from source err:%v", err)
}
// 取出值存入多级缓存
_ = v
return nil
}
// 等待数据加载完成
select {
case <-ch:
// 未取到结果时,再次获取
return c.getFromSource(ctx, key, fn)
}
}

6
cache/readme.md vendored
View File

@ -11,3 +11,9 @@
3. 缓存穿透;从数据源中未找到数据时,在缓存中缓存空值。
4. 缓存雪崩;为防止缓存雪崩将资源放入缓存时,对过期时间添加一个随机过期时间,防止缓存同时过期。
5. 自动续期;当访问二级缓存时对使用的资源进行延期。
## 使用方式
```go
Cache.Get(key, dist, func() (bool,error){}, options func(){})
```

View File

@ -0,0 +1,24 @@
package cleanupguard
import "sync"
type CleanupGuard struct {
enable bool
fn func()
mutex sync.Mutex
}
// 新建清理
func NewCleanupGuard(fn func()) CleanupGuard {
return CleanupGuard{fn: fn, enable: true}
}
func (g *CleanupGuard) Enable() {
g.mutex.Lock()
defer g.mutex.Unlock()
g.enable = true
}
func (g *CleanupGuard) Run() {
g.fn()
}

View File

@ -0,0 +1,17 @@
package compiledbuffer
import (
"testing"
"github.com/dlclark/regexp2"
)
func TestCom(t *testing.T) {
regex, err := regexp2.Compile(`^\d{11}[;](?!(37|38))\d{2}\d{6}$`, regexp2.None)
if err != nil {
t.Fatal(err)
}
t.Log(regex.MatchString("14610522152;37764800"))
t.Log(regex.MatchString("14610522152;33764800"))
}

View File

@ -42,7 +42,8 @@ func (z *zipPackage) Write(out *os.File) error {
zipWriter := zip.NewWriter(out)
defer zipWriter.Close()
for _, f := range z.files {
files := z.files
for _, f := range files {
fileWriter, err := zipWriter.Create(f.name)
if err != nil {
return err

View File

@ -7,6 +7,7 @@ import (
const (
layoutDate = "2006-01-02"
layoutTime = "2006-01-02 15:04:05"
layoutTimeMilli = "2006-01-02 15:04:05.000"
layoutChineseDate = "2006年01月02日"
layoutChineseTime = "2006年01月02日 15:04:05"
)

3
db/readme.md Normal file
View File

@ -0,0 +1,3 @@
# 数据访问层,创建
使用gorm作为数据访问层

22
go.mod
View File

@ -3,8 +3,9 @@ module github.com/charlienet/go-mixed
go 1.18
require (
github.com/bits-and-blooms/bitset v1.2.2
github.com/bits-and-blooms/bitset v1.3.0
github.com/cespare/xxhash/v2 v2.1.2
github.com/go-playground/universal-translator v0.18.0
github.com/json-iterator/go v1.1.12
github.com/shopspring/decimal v1.3.1
github.com/spaolacci/murmur3 v1.1.0
@ -14,9 +15,16 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
github.com/jonboulle/clockwork v0.3.0 // indirect
github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect
github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/tebeka/strftime v0.1.5 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
@ -24,13 +32,15 @@ require (
github.com/allegro/bigcache/v3 v3.0.2
github.com/antonfisher/nested-logrus-formatter v1.3.1
github.com/coocood/freecache v1.2.1
github.com/dlclark/regexp2 v1.7.0
github.com/go-redis/redis/v8 v8.11.5
github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.2
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/exp v0.0.0-20220608143224-64259d1afd70
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)

57
go.sum
View File

@ -7,6 +7,8 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UME
github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA=
github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk=
github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bits-and-blooms/bitset v1.3.0 h1:h7mv5q31cthBTd7V4kLAZaIThj1e8vPGcSqpPue9KVI=
github.com/bits-and-blooms/bitset v1.3.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@ -19,10 +21,20 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 h1:Ghm4eQYC0nEPnSJdVkTrXpu9KtoVCSo1hg7mtI7G9KU=
github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -41,10 +53,18 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 h1:IPJ3dvxmJ4uczJe5YQdrYB16oTJlGSC/OyZDqUk9xX4=
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag=
github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg=
github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 h1:0iQektZGS248WXmGIYOwRXSQhD4qn3icjMpuxwO7qlo=
github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570/go.mod h1:BLt8L9ld7wVsvEWQbuLrUZnCMnUmLZ+CGDzKtclrTlE=
github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f h1:sgUSP4zdTUZYZgAGGtN5Lxk92rK+JUFOwf+FT99EEI4=
github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f/go.mod h1:UGmTpUd3rjbtfIpwAPrcfmGf/Z1HS95TATB+m57TPB8=
github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 h1:Bvq8AziQ5jFF4BHGAEDSqwPW1NJS3XshxbRCxtjFAZc=
github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042/go.mod h1:TPpsiPUEh0zFL1Snz4crhMlBe60PYxRHr5oFF3rRYg0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -62,23 +82,34 @@ github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5g
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/tebeka/strftime v0.1.5 h1:1NQKN1NiQgkqd/2moD6ySP/5CoZQsKa1d3ZhJ44Jpmg=
github.com/tebeka/strftime v0.1.5/go.mod h1:29/OidkoWHdEKZqzyDLUyC+LmgDgdHo4WAFCDT7D/Ig=
github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=
github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20220608143224-64259d1afd70 h1:8uGxpY2cLF9H/NSHUiEWUIBZqIcsMzMWIMPCCUkyYgc=
golang.org/x/exp v0.0.0-20220608143224-64259d1afd70/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys=
golang.org/x/exp v0.0.0-20220713135740-79cabaa25d75 h1:x03zeu7B2B11ySp+daztnwM5oBJ/8wGUSqrwcw9L0RA=
golang.org/x/exp v0.0.0-20220713135740-79cabaa25d75/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -98,11 +129,16 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -129,6 +165,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -21,13 +21,13 @@ var _ crypto.Signer = &hashComparer{}
type HMacFunc func(key, msg []byte) bytesconv.BytesResult
var hmacFuncs = map[string]HMacFunc{
"HmacMD5": Md5,
"HmacSHA1": Sha1,
"HmacSHA224": Sha224,
"HmacSHA256": Sha256,
"HmacSHA384": Sha384,
"HmacSHA512": Sha512,
"HmacSM3": Sm3,
"HMACMD5": Md5,
"HMACSHA1": Sha1,
"HMACSHA224": Sha224,
"HMACSHA256": Sha256,
"HMACSHA384": Sha384,
"HMACSHA512": Sha512,
"HMACSM3": Sm3,
}
type hashComparer struct {
@ -63,7 +63,7 @@ func ByName(name string) (HMacFunc, error) {
return f, nil
}
return nil, errors.New("Unsupported hash functions")
return nil, errors.New("Unsupported hash function:" + name)
}
func Md5(key, msg []byte) bytesconv.BytesResult { return sum(md5.New, key, msg) }

124
ip_range/ip_range.go Normal file
View File

@ -0,0 +1,124 @@
package iprange
import (
"fmt"
"net/netip"
"regexp"
"strconv"
"strings"
)
var maskPattern = regexp.MustCompile(`\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b`)
type IpRange struct {
segments []ipSegment
}
type ipSegment interface {
Contains(netip.Addr) bool
}
type singleIp struct {
ip netip.Addr
}
func (i *singleIp) Contains(ip netip.Addr) bool {
return i.ip.Compare(ip) == 0
}
type prefixSegments struct {
prefix netip.Prefix
}
func (i *prefixSegments) Contains(ip netip.Addr) bool {
return i.prefix.Contains(ip)
}
type rangeSegment struct {
start netip.Addr
end netip.Addr
}
func (r *rangeSegment) Contains(ip netip.Addr) bool {
return ip.Compare(r.start) >= 0 && ip.Compare(r.end) <= 0
}
// IP范围判断支持以下规则:
// 单IP地址如 192.168.100.2
// IP范围, 如 192.168.100.120-192.168.100.150
// 掩码模式,如 192.168.2.0/24
func NewRange(ip ...string) (*IpRange, error) {
seg := make([]ipSegment, 0, len(ip))
for _, i := range ip {
if s, err := createSegment(i); err != nil {
return nil, err
} else {
seg = append(seg, s)
}
}
return &IpRange{segments: seg}, nil
}
func (r *IpRange) Contains(ip string) bool {
addr, err := netip.ParseAddr(ip)
if err != nil {
return false
}
for _, v := range r.segments {
if v.Contains(addr) {
return true
}
}
return false
}
func createSegment(ip string) (ipSegment, error) {
switch {
case strings.Contains(ip, "-"):
ips := strings.Split(ip, "-")
if len(ips) != 2 {
return nil, fmt.Errorf("IP范围定义错误:%s", ip)
}
start, err := netip.ParseAddr(ips[0])
if err != nil {
return nil, err
}
end, err := netip.ParseAddr(ips[1])
if err != nil {
return nil, err
}
return &rangeSegment{
start: start,
end: end,
}, nil
case strings.Contains(ip, "/"):
sec := strings.Split(ip, "/")
ip := sec[0]
mask := sec[1]
if maskPattern.MatchString(mask) {
mask = strconv.Itoa(MaskToBits(mask))
}
if prefix, err := netip.ParsePrefix(ip + "/" + mask); err != nil {
return nil, err
} else {
return &prefixSegments{prefix: prefix}, nil
}
default:
i, err := netip.ParseAddr(ip)
if err != nil {
return nil, fmt.Errorf("格式错误, 不是有效的IP地址:%s", ip)
}
return &singleIp{ip: i}, nil
}
}

130
ip_range/ip_range_test.go Normal file
View File

@ -0,0 +1,130 @@
package iprange
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSingleErrorIP(t *testing.T) {
values := []string{
"192.168.01",
"::",
}
for _, v := range values {
r, err := NewRange(v)
t.Log(r, err)
}
}
func TestSingleIp(t *testing.T) {
r, err := NewRange("192.168.0.1")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.0.1"))
assert.False(t, r.Contains("192.168.0.123"))
}
func TestSinglePrefix(t *testing.T) {
r, err := NewRange("192.168.2.100/32")
if err != nil {
t.Fatal(err)
}
assert.False(t, r.Contains("192.168.2.56"))
assert.True(t, r.Contains("192.168.2.100"))
assert.False(t, r.Contains("192.168.2.130"))
}
func TestAllIp(t *testing.T) {
r, err := NewRange("0.0.0.0/0")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.2.100"))
assert.True(t, r.Contains("192.3.2.100"))
assert.True(t, r.Contains("192.65.2.100"))
assert.True(t, r.Contains("172.168.2.100"))
assert.True(t, r.Contains("8.8.8.8"))
assert.True(t, r.Contains("114.114.114.114"))
}
func TestPrefix(t *testing.T) {
r, err := NewRange("192.168.2.0/24")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.2.12"))
assert.True(t, r.Contains("192.168.2.162"))
assert.False(t, r.Contains("192.168.3.162"))
}
func TestPrefix2(t *testing.T) {
r, err := NewRange("192.168.15.0/21")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.8.10"))
assert.True(t, r.Contains("192.168.14.162"))
assert.False(t, r.Contains("192.168.3.162"))
assert.False(t, r.Contains("192.168.2.162"))
}
func TestDotMask(t *testing.T) {
r, err := NewRange("192.168.15.0/255.255.248.0")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.8.10"))
assert.True(t, r.Contains("192.168.14.162"))
assert.False(t, r.Contains("192.168.3.162"))
assert.False(t, r.Contains("192.168.2.162"))
}
func TestRange(t *testing.T) {
r, err := NewRange("192.168.2.20-192.168.2.30")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("192.168.2.20"))
assert.True(t, r.Contains("192.168.2.21"))
assert.True(t, r.Contains("192.168.2.30"))
assert.False(t, r.Contains("192.168.2.10"))
assert.False(t, r.Contains("192.168.2.31"))
}
func TestLocalhost(t *testing.T) {
r, err := NewRange("::1")
if err != nil {
t.Fatal(err)
}
assert.True(t, r.Contains("::1"))
}
func TestNetIP(t *testing.T) {
addr, err := netip.ParseAddr("192.168.2.10")
if err != nil {
t.Fatal(err)
}
t.Log(netip.MustParseAddr("192.168.2.4").Compare(addr))
t.Log(netip.MustParseAddr("192.168.2.10").Compare(addr))
t.Log(netip.MustParseAddr("192.168.2.11").Compare(addr))
prefix := netip.MustParsePrefix("192.168.2.0/24")
t.Log(prefix.Contains(netip.MustParseAddr("192.168.2.53")))
t.Log(prefix.Contains(netip.MustParseAddr("192.168.3.53")))
}

34
ip_range/mask_bits.go Normal file
View File

@ -0,0 +1,34 @@
package iprange
import "strings"
var maskBits = map[string]int{
"255": 8,
"254": 7,
"252": 6,
"248": 5,
"240": 4,
"224": 3,
"192": 2,
"128": 1,
"0": 0,
}
func MaskToBits(mask string) int {
bits := 0
secs := strings.Split(mask, ".")
if len(secs) != 4 {
panic("the mask is incorrect")
}
for _, s := range secs {
if v, ok := maskBits[s]; ok {
bits += v
} else {
panic("the mask is incorrect")
}
}
return bits
}

View File

@ -0,0 +1,26 @@
package iprange
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestMaskToBits(t *testing.T) {
masks := []struct {
mask string
expect int
}{
{"255.255.255.0", 24},
{"255.255.248.0", 21},
{"255.255.192.0", 18},
{"255.255.255.192", 26},
}
for _, m := range masks {
bits := MaskToBits(m.mask)
assert.Equal(t, m.expect, bits, fmt.Sprintf("IP:%s 掩码位数错误。", m.mask))
}
}

View File

@ -5,6 +5,9 @@ package json
import "encoding/json"
func RegisterFuzzyDecoders() {
}
var (
Marshal = json.Marshal
Unmarshal = json.Unmarshal

View File

@ -0,0 +1,41 @@
package locker
type ChanLocker interface {
Get(key string) (ch <-chan int, ok bool)
Release(key string)
}
type chanSourceLock struct {
m RWLocker
content map[string]chan int
}
func (s *chanSourceLock) Get(key string) (ch <-chan int, ok bool) {
s.m.RLock()
ch, ok = s.content[key]
s.m.RUnlock()
if ok {
return
}
s.m.Lock()
ch, ok = s.content[key]
if ok {
s.m.Unlock()
return
}
s.content[key] = make(chan int)
ch = s.content[key]
ok = true
s.m.Unlock()
return
}
func (s *chanSourceLock) Release(key string) {
s.m.Lock()
ch, ok := s.content[key]
if ok {
close(ch)
delete(s.content, key)
}
s.m.Unlock()
}

View File

@ -2,53 +2,93 @@ package locker
import (
"fmt"
"sync/atomic"
)
// 带计数器锁
type countLocker struct {
Locker
Count int32
}
// 资源锁
type SourceLocker struct {
m RWLocker
locks map[string]Locker
locks map[string]*countLocker
}
func NewSourceLocker() *SourceLocker {
return &SourceLocker{
m: NewRWLocker(),
locks: make(map[string]Locker),
locks: make(map[string]*countLocker),
}
}
func (s *SourceLocker) Lock(key string) {
s.m.RLock()
l, ok := s.locks[key]
s.m.RUnlock()
if ok {
s.m.RUnlock()
atomic.AddInt32(&l.Count, 1)
l.Lock()
fmt.Println("加锁")
} else {
s.m.RUnlock()
// 加锁,再次检查是否已经具有锁
s.m.Lock()
new := NewLocker()
s.locks[key] = new
if l2, ok := s.locks[key]; ok {
s.m.Unlock()
l2.Lock()
fmt.Println("二次检查加锁")
} else {
new := NewLocker()
s.locks[key] = &countLocker{Locker: new, Count: 1}
s.m.Unlock()
fmt.Printf("新锁准备加锁:%p\n", new)
new.Lock()
fmt.Println("初始加锁")
}
}
}
func (s *SourceLocker) Unlock(key string) {
s.m.Lock()
if l, ok := s.locks[key]; ok {
atomic.AddInt32(&l.Count, -1)
fmt.Printf("解锁%p\n", l)
l.Unlock()
// delete(s.locks, key)
fmt.Println("解锁")
if l.Count == 0 {
delete(s.locks, key)
}
}
s.m.Unlock()
}
func (s *SourceLocker) TryLock(key string) bool {
return false
// 加读锁
s.m.RLock()
l, ok := s.locks[key]
if ok {
ret := l.TryLock()
s.m.RUnlock()
return ret
} else {
s.m.RUnlock()
s.m.Lock()
new := NewLocker()
s.locks[key] = &countLocker{Locker: new, Count: 1}
s.m.Unlock()
return new.TryLock()
}
}

View File

@ -3,10 +3,15 @@ package locker
import (
"sync"
"testing"
"time"
)
var sourcekey = "u-0001"
func TestTryLock(t *testing.T) {
}
func TestSourceLocker(t *testing.T) {
l := NewSourceLocker()
@ -27,6 +32,32 @@ func TestSourceLocker(t *testing.T) {
wg.Wait()
t.Log("n:", n)
t.Logf("%+v", l)
}
func TestSourceTryLock(t *testing.T) {
c := 5
n := 0
wg := new(sync.WaitGroup)
wg.Add(c)
l := NewSourceLocker()
for i := 0; i < c; i++ {
go func() {
defer wg.Done()
if l.TryLock(sourcekey) {
n++
time.Sleep(time.Second)
l.Unlock(sourcekey)
}
}()
}
wg.Wait()
t.Log("n:", n)
t.Logf("%+v", l)
}
func BenchmarkSourceLocker(b *testing.B) {

View File

@ -1,5 +1,7 @@
package logx
import "io"
var std = defaultLogger()
func StandardLogger() Logger {
@ -34,4 +36,5 @@ type Logger interface {
Println(args ...any)
Print(args ...any)
Printf(format string, args ...any)
Writer() io.Writer
}

View File

@ -1,3 +1,23 @@
package logx
import (
"io"
"os"
)
type Rotate int
const (
None Rotate = iota // 不分割日志
Size // 按大小分割
Date // 按日期分割
)
type OutputOptions struct {
LogrusOutputOptions
}
func WithFile(filename string) (io.Writer, error) {
mode := os.FileMode(0644)
return os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, mode)
}

1
logx/logger.go Normal file
View File

@ -0,0 +1 @@
package logx

View File

@ -1,16 +0,0 @@
package logx
type loggerBuilder struct {
}
func NewBuilder() *loggerBuilder {
return &loggerBuilder{}
}
func (b *loggerBuilder) WithLogrus() *loggerBuilder {
return b
}
func (b *loggerBuilder) WithLogger() *loggerBuilder {
return b
}

View File

@ -2,14 +2,8 @@ package logx_test
import (
"testing"
"github.com/charlienet/go-mixed/logx"
)
func TestBuilder(t *testing.T) {
logger := logx.NewBuilder().
WithLogrus().
WithLogger()
_ = logger
}

View File

@ -1,6 +1,8 @@
package logx
import (
"io"
"github.com/sirupsen/logrus"
)
@ -37,3 +39,7 @@ func (l *logrusWrpper) WithFields(fields Fields) Logger {
return l
}
func (l *logrusWrpper) Writer() io.Writer {
return l.Entry.Writer()
}

View File

@ -6,6 +6,7 @@ import (
"runtime"
nested "github.com/antonfisher/nested-logrus-formatter"
"github.com/sirupsen/logrus"
)
const defaultTimestampFormat = "2006-01-02 15:04:05.000"
@ -14,6 +15,14 @@ type NestedFormatterOption struct {
Color bool
}
func NewJsonFormatter() logrus.Formatter {
return &logrus.JSONFormatter{}
}
func NewTextFOrmatter() logrus.Formatter {
return &logrus.TextFormatter{}
}
func NewNestedFormatter(option NestedFormatterOption) *nested.Formatter {
return &nested.Formatter{
TimestampFormat: defaultTimestampFormat,

View File

@ -4,6 +4,7 @@ import (
"io"
"log"
"os"
"time"
"github.com/charlienet/go-mixed/fs"
"github.com/sirupsen/logrus"
@ -32,6 +33,7 @@ type LogrusOutputOptions struct {
}
type LogrusBackupOptions struct {
BackupType Rotate // 分割类型
MaxSize int // 默认大小100M
MaxAge int // 备份保留天数
MaxBackups int // 备份保留数量
@ -66,6 +68,7 @@ func WithFormatter(formatter logrus.Formatter) logrusOption {
}
func WithOutput(options LogrusOutputOptions) logrusOption {
_ = time.Now()
return func(l *logrus.Logger) {
var writer io.Writer
switch {

View File

@ -39,9 +39,3 @@ func TestLevel(t *testing.T) {
// logger.SetLevel(l)
logger.Info("bcdefg")
}
func TestMutiWriter(t *testing.T) {
l := NewLogger().AppendLogger()
_ = l
}

View File

@ -1,12 +0,0 @@
package logx
type mutiLogger struct {
}
func NewLogger() *mutiLogger {
return &mutiLogger{}
}
func (w *mutiLogger) AppendLogger() Logger {
return nil
}

26
logx/readme.md Normal file
View File

@ -0,0 +1,26 @@
# 日志记录组件
日志分割及备份
日志可按照日期或大小进行分割,保留的历史日志文件数量由备份数量决定。
1. 按天拆分每天生成新的日志文件名称。格式为file.yyyy-mm-dd.log 其中file和log为配置的日志文件名称。
2. 按大小拆分使用lumberjack组件对日志文件进行分割。
3. 按时间间隔拆分,日志文件按照指定的间隔拆分,
日志输出流
支持控制台和文件输出,可扩展输出组件
``` golang
logx.NewLogger(
WithLevel("debug"),
WithFormatter(),
WithConsole(),
WithRoateBySize(FileRoateSize{
MaxSize
MaxAge
MaxBackups
}),
WithRoateByDate("filename", MaxAge, MaxBackups),
WithFile("filename"))
```

View File

@ -0,0 +1,22 @@
package logx
import (
"io"
)
// ensure we always implement io.WriteCloser
var _ io.WriteCloser = (*rotateDateWriter)(nil)
type rotateDateWriter struct {
MaxAge int
MaxBackups int
}
func (l *rotateDateWriter) Write(p []byte) (n int, err error) {
return 0, nil
}
func (l *rotateDateWriter) Close() error {
return nil
}

View File

@ -0,0 +1,3 @@
package logx
// 按大小分割的日志记录器

View File

@ -0,0 +1,27 @@
package logx
import (
"path/filepath"
"testing"
"time"
rotatelogs "github.com/lestrrat/go-file-rotatelogs"
)
func TestNewWriter(t *testing.T) {
t.Log(filepath.Abs("logs"))
logf, err := rotatelogs.New("logs/aaaa.%Y%m%d.log",
rotatelogs.WithMaxAge(24*time.Hour),
rotatelogs.WithRotationTime(time.Hour))
if err != nil {
t.Fatal(err)
}
defer logf.Close()
t.Log(logf.CurrentFileName())
_, err = logf.Write([]byte("abaccad"))
t.Log(err)
}

View File

@ -62,6 +62,10 @@ func (m *rw_map[K, V]) ToMap() map[K]V {
return m.m.ToMap()
}
func (m *rw_map[K, V]) Shrink() map[K]V {
return m.m.ToMap()
}
func (m *rw_map[K, V]) Exist(key K) bool {
return m.m.Exist(key)
}

View File

@ -88,7 +88,8 @@ func (m *sorted_map[K, V]) Iter() <-chan *Entry[K, V] {
}
func (m *sorted_map[K, V]) ForEach(f func(K, V) bool) {
for _, k := range m.keys {
keys := m.keys[:]
for _, k := range keys {
if v, ok := m.Get(k); ok {
if f(k, v) {
break

View File

@ -14,3 +14,10 @@ func Max[T constraints.Ordered](v1, v2 T) T {
func Min[T constraints.Ordered](v1, v2 T) T {
return expr.If(v1 < v2, v1, v2)
}
func abs(n int64) int64 {
y := n >> 63
return (n ^ y) - y
}

81
panic/panic.go Normal file
View File

@ -0,0 +1,81 @@
package panic
import (
"context"
"fmt"
"runtime/debug"
)
type Panic struct {
R any
Stack []byte
}
func (p Panic) String() string {
return fmt.Sprintf("%v\n%s", p.R, p.Stack)
}
type PanicGroup struct {
panics chan Panic // 致命错误通知
dones chan int // 协程完成通知
jobs chan int // 并发数量
jobN int32 // 工作协程数量
}
func NewPanicGroup(maxConcurrent int) *PanicGroup {
return &PanicGroup{
panics: make(chan Panic, 8),
dones: make(chan int, 8),
jobs: make(chan int, maxConcurrent),
}
}
func (g *PanicGroup) Go(f func()) *PanicGroup {
g.jobN++
go func() {
g.jobs <- 1
defer func() {
<-g.jobs
// go 语言只能在自己的协程中捕获自己的 panic
// 如果不处理,整个*进程*都会退出
if r := recover(); r != nil {
g.panics <- Panic{R: r, Stack: debug.Stack()}
// 如果发生 panic 就不再通知 Wait() 已完成
// 不然就可能出现 g.jobN 为 0 但 g.panics 非空
// 的情况,此时 Wait() 方法需要在正常结束的分支
// 中再额外检查是否发生了 panic非常麻烦
return
}
g.dones <- 1
}()
f()
}()
return g
}
func (g *PanicGroup) Wait(ctx context.Context) error {
if g.jobN == 0 {
panic("no job to wait")
}
for {
select {
case <-g.dones: // 协程正常结束
g.jobN--
if g.jobN == 0 {
return nil
}
case p := <-g.panics: // 协程有 panic
panic(p)
case <-ctx.Done():
// 整个 ctx 结束,超时或者调用方主动取消
// 子协程应该共用该 ctx都会收到相同的结束信号
// 不需要在这里再去通知各协程结束(实现起来也麻烦)
return ctx.Err()
}
}
}

32
panic/panic_test.go Normal file
View File

@ -0,0 +1,32 @@
package panic
import (
"context"
"fmt"
"testing"
"time"
)
func TestPanic(t *testing.T) {
defer func() {
t.Log("捕捉异常")
if e := recover(); e != nil {
if err, ok := e.(error); ok {
t.Log(err.Error())
}
t.Log("格式化:", e)
}
}()
g := NewPanicGroup(10)
g.Go(func() {
panic("1243")
})
if err := g.Wait(context.Background()); err != nil {
panic(err)
}
time.Sleep(1 * time.Second)
fmt.Println("这条消息可打印")
}

View File

@ -6,37 +6,62 @@ import (
"fmt"
"strings"
"github.com/charlienet/go-mixed/locker"
"golang.org/x/exp/constraints"
)
var _ Set[string] = &hash_set[string]{}
type hash_set[T constraints.Ordered] map[T]struct{}
type hash_set[T constraints.Ordered] struct {
m map[T]struct{}
lock locker.RWLocker
}
func NewHashSet[T constraints.Ordered](values ...T) *hash_set[T] {
set := make(hash_set[T], len(values))
set := hash_set[T]{
m: make(map[T]struct{}, len(values)),
lock: locker.EmptyLocker,
}
set.Add(values...)
return &set
}
func (s *hash_set[T]) WithSync() *hash_set[T] {
s.lock = locker.NewRWLocker()
return s
}
func (s hash_set[T]) Add(values ...T) {
s.lock.Lock()
defer s.lock.Unlock()
for _, v := range values {
s[v] = struct{}{}
s.m[v] = struct{}{}
}
}
func (s hash_set[T]) Remove(v T) {
delete(s, v)
s.lock.Lock()
defer s.lock.Unlock()
delete(s.m, v)
}
func (s hash_set[T]) Contains(value T) bool {
_, ok := s[value]
s.lock.RLock()
defer s.lock.RUnlock()
_, ok := s.m[value]
return ok
}
func (s hash_set[T]) ContainsAny(values ...T) bool {
s.lock.RLock()
defer s.lock.RUnlock()
for _, v := range values {
if _, ok := s[v]; ok {
if _, ok := s.m[v]; ok {
return true
}
}
@ -45,8 +70,11 @@ func (s hash_set[T]) ContainsAny(values ...T) bool {
}
func (s hash_set[T]) ContainsAll(values ...T) bool {
s.lock.RLock()
defer s.lock.RUnlock()
for _, v := range values {
if _, ok := s[v]; !ok {
if _, ok := s.m[v]; !ok {
return false
}
}
@ -64,7 +92,7 @@ func (s hash_set[T]) Desc() Set[T] {
func (s hash_set[T]) copyToSorted() Set[T] {
orderd := NewSortedSet[T]()
for k := range s {
for k := range s.m {
orderd.Add(k)
}
@ -78,7 +106,7 @@ func (s *hash_set[T]) Clone() *hash_set[T] {
}
func (s hash_set[T]) Iterate(fn func(value T)) {
for v := range s {
for v := range s.m {
fn(v)
}
}
@ -93,17 +121,17 @@ func (s hash_set[T]) ToSlice() []T {
}
func (s hash_set[T]) IsEmpty() bool {
return len(s) == 0
return len(s.m) == 0
}
func (s hash_set[T]) Size() int {
return len(s)
return len(s.m)
}
func (s hash_set[T]) MarshalJSON() ([]byte, error) {
items := make([]string, 0, s.Size())
for ele := range s {
for ele := range s.m {
b, err := json.Marshal(ele)
if err != nil {
return nil, err
@ -135,8 +163,8 @@ func (s hash_set[T]) UnmarshalJSON(b []byte) error {
}
func (s hash_set[T]) String() string {
l := make([]string, 0, len(s))
for k := range s {
l := make([]string, 0, len(s.m))
for k := range s.m {
l = append(l, fmt.Sprint(k))
}

View File

@ -1,17 +1,19 @@
package snowflake
import (
"fmt"
"log"
"sync"
"time"
)
// 雪花算法默认起始时间 2020-01-01
const defaultStarTimestamp = 1579536000
// 雪花算法默认起始时间 2022-01-01
const defaultStarTimestamp = 1640966400000
const (
MachineIdBits = uint(8) //机器id所占的位数
SequenceBits = uint(12) //序列所占的位数
//MachineIdMax = int64(-1 ^ (-1 << MachineIdBits)) //支持的最大机器id数量
MachineIdMax = int64(-1 ^ (-1 << MachineIdBits)) //支持的最大机器id数量
SequenceMask = int64(-1 ^ (-1 << SequenceBits)) //
MachineIdShift = SequenceBits //机器id左移位数
TimestampShift = SequenceBits + MachineIdBits //时间戳左移位数
@ -30,28 +32,42 @@ type snowflake struct {
}
func CreateSnowflake(machineId int64) SnowFlake {
timeBits := 63 - MachineIdBits - SequenceBits
maxTime := time.UnixMilli(defaultStarTimestamp + (int64(-1 ^ (-1 << timeBits))))
log.Println("最大可用时间:", maxTime)
return &snowflake{
startTimeStamp: defaultStarTimestamp,
machineId: machineId,
machineId: machineId & MachineIdMax,
}
}
// 组织方式 时间戳-机器码-序列号
func (s *snowflake) GetId() int64 {
// 生成序列号规则
// 检查当前生成时间与上次生成时间对比
// 如等于上次生成时间,检查是否已经达到序列号的最大值,如已达到等待下一个时间点并且设置序列号为零。
// 如不相等则序列号自增
s.Lock()
defer s.Unlock()
now := time.Now().UnixNano() / 1e6
if s.timestamp == now {
s.sequence = (s.sequence + 1) & SequenceMask
if s.sequence == 0 {
now := time.Now().UnixMilli()
if s.timestamp == now && s.sequence == 0 {
fmt.Println(time.Now().Format("2006-01-02 15:04:05.000"), "下一个时间点")
for now <= s.timestamp {
now = time.Now().UnixNano() / 1e6
now = time.Now().UnixMilli()
}
}
} else {
s.sequence = 0
}
s.timestamp = now
s.sequence = (s.sequence + 1) & SequenceMask
log.Println("时间戳:", now-s.startTimeStamp)
log.Println("时间差:", time.Now().Sub(time.UnixMilli(defaultStarTimestamp)))
r := (now-s.startTimeStamp)<<TimestampShift | (s.machineId << MachineIdShift) | (s.sequence)
return r
}

View File

@ -1,6 +1,15 @@
package snowflake
import "testing"
import (
"testing"
"github.com/charlienet/go-mixed/sets"
)
func TestGet(t *testing.T) {
s := CreateSnowflake(2)
t.Log(s.GetId())
}
func TestGetId(t *testing.T) {
s := CreateSnowflake(22)
@ -8,3 +17,46 @@ func TestGetId(t *testing.T) {
t.Log(s.GetId())
}
}
func TestMutiGetId(t *testing.T) {
s := CreateSnowflake(11)
for i := 0; i < 100000; i++ {
s.GetId()
}
}
func TestMutiConflict(t *testing.T) {
set := sets.NewHashSet[int64]()
s := CreateSnowflake(11)
for i := 0; i < 10000000; i++ {
id := s.GetId()
if set.Contains(id) {
t.Fatal("失败,生成重复数据")
}
set.Add(id)
}
}
func BenchmarkGetId(b *testing.B) {
s := CreateSnowflake(11)
for i := 0; i < b.N; i++ {
s.GetId()
}
}
func BenchmarkMutiGetId(b *testing.B) {
s := CreateSnowflake(11)
set := sets.NewHashSet[int64]().WithSync()
b.RunParallel(func(p *testing.PB) {
for i := 0; p.Next(); i++ {
id := s.GetId()
if set.Contains(id) {
b.Fatal("标识重复", id)
}
set.Add(id)
}
})
}

View File

@ -40,7 +40,9 @@ func (s *mapSorter[T]) Desc() *mapSorter[T] {
func (s *mapSorter[T]) Join(sep string, f func(k string, v T) string) string {
slice := make([]string, 0, len(s.m))
for _, k := range s.keys {
keys := s.keys[:]
for _, k := range keys {
slice = append(slice, f(k, s.m[k]))
}
@ -53,7 +55,9 @@ func (s *mapSorter[T]) Keys() []string {
func (s *mapSorter[T]) Values() []T {
ret := make([]T, 0, len(s.m))
for _, k := range s.keys {
keys := s.keys[:]
for _, k := range keys {
ret = append(ret, s.m[k])
}

View File

@ -36,7 +36,8 @@ func (s *Struct) Kind() reflect.Kind {
func (s *Struct) Names() []string {
names := make([]string, len(s.fields))
for i, f := range s.fields {
fields := s.fields[:]
for i, f := range fields {
names[i] = f.name
}
@ -45,7 +46,9 @@ func (s *Struct) Names() []string {
func (s *Struct) Values() []any {
values := make([]any, 0, len(s.fields))
for _, fi := range s.fields {
fields := s.fields[:]
for _, fi := range fields {
v := s.value.FieldByName(fi.name)
values = append(values, v.Interface())
}
@ -54,7 +57,8 @@ func (s *Struct) Values() []any {
}
func (s *Struct) IsZero() bool {
for _, fi := range s.fields {
fields := s.fields[:]
for _, fi := range fields {
source := s.value.FieldByName(fi.name)
if !source.IsZero() {
return false
@ -66,7 +70,9 @@ func (s *Struct) IsZero() bool {
func (s *Struct) ToMap() map[string]any {
m := make(map[string]any, len(s.fields))
for _, fi := range s.fields {
fields := s.fields[:]
for _, fi := range fields {
source := s.value.FieldByName(fi.name)
if fi.shouldIgnore(source) {
continue
@ -109,7 +115,8 @@ func (s *Struct) Copy(dest any) error {
}
func (s *Struct) getByName(name string) (field, bool) {
for i := range s.fields {
fields := s.fields[:]
for i := range fields {
f := s.fields[i]
if f.name == name {
return f, true

View File

@ -3,6 +3,7 @@ package tests
import (
"bytes"
"fmt"
"net"
"testing"
)
@ -37,3 +38,16 @@ func BenchmarkStringSplice(b *testing.B) {
}
})
}
func TestIPSegment(t *testing.T) {
i, n, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
t.Fatal(err)
}
t.Log(i, n)
address := net.ParseIP("192.168.0.2")
t.Log(n.Contains(address))
}

View File

@ -0,0 +1,7 @@
package validatortranslation
import (
ut "github.com/go-playground/universal-translator"
)
var Trans ut.Translator

View File

@ -0,0 +1 @@
package workerpool