go/go-mixed
1
0
Fork 0
mirror of https://github.com/charlienet/go-mixed.git synced 2025-07-18 00:22:41 +08:00
Code Issues Projects Releases Wiki Activity

更新ID生成器

Browse Source
This commit is contained in:
charlie
2023-10-26 14:42:23 +08:00
parent 6647f96978
commit 0d124c0b79
17 changed files with 1079 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
idGenerator/assets/双缓存号段分配.webp Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

72
idGenerator/buffer.go Normal file
View File

@ -0,0 +1,72 @@
package idgenerator
import (
"sync"
"time"
"github.com/charlienet/go-mixed/idGenerator/store"
)
type obtainFunc func() (*store.Segment, error)
type doubleBuffer struct {
current *store.Segment // 当前
backup *store.Segment // 备用
obtain obtainFunc // 数据段获取函数
inFill bool // 备用缓冲填充中
isReadly bool // 备用缓冲区填充完成
mu sync.Mutex
}
func newDoubleBuffer(obtainFunc obtainFunc) *doubleBuffer {
b := &doubleBuffer{obtain: obtainFunc}
b.current, _ = b.obtain()
return b
}
func (b *doubleBuffer) allot() (int64, bool) {
if !b.inFill && b.current.IsEnding() {
go b.full() // 填充备用缓冲
}
// 缓冲区耗尽时切换
if b.current.IsEmpty() {
// 检查备用缓冲是否已经填充完成，已完成时切换，否则等待
for !b.isReadly {
time.Sleep(time.Microsecond * 100)
}
b.switchBuf()
}
return b.current.Allot(), b.current.Reback()
}
func (b *doubleBuffer) full() {
b.mu.Lock()
defer b.mu.Unlock()
if !b.inFill {
var err error
b.inFill = true
b.backup, err = b.obtain()
if err != nil {
println("填充失败:", err.Error())
panic(err)
}
b.isReadly = true
}
}
func (b *doubleBuffer) switchBuf() {
b.mu.Lock()
defer b.mu.Unlock()
if b.isReadly {
b.current, b.backup = b.backup, b.current
b.inFill = false
b.isReadly = false
}
}

25
idGenerator/buffer_test.go Normal file
View File

@ -0,0 +1,25 @@
package idgenerator
import (
"testing"
"github.com/charlienet/go-mixed/idGenerator/store"
"github.com/charlienet/go-mixed/redis"
"github.com/charlienet/go-mixed/tests"
)
func TestBufferAlloc(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
f := func() (*store.Segment, error) {
return store.NewRedisStore("sss", rdb).Assign(3, 99, 10)
}
b := newDoubleBuffer(f)
for i := 0; i < 80; i++ {
t.Log(b.allot())
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}

132
idGenerator/formater.go Normal file
View File

@ -0,0 +1,132 @@
package idgenerator
import (
"errors"
"math"
"strconv"
"time"
)
const (
YYYYMMDDHHmmss = "20060102150405"
YYYYMMDDHHmm = "200601021504"
YYYYMMDDHH = "2006010215"
YYYYMMDD = "20060102"
)
type Layout int
const (
Binary Layout = iota
Decimal
)
type formater interface {
MaxinalMachineCode() int64
MaximalSequence() int64
Format(machine, serial int64, reback bool) Id
}
type Id int64
func (i Id) String() string {
if i == 0 {
return ""
}
return strconv.FormatInt(int64(i), 10)
}
func (i Id) Int64() int64 {
return int64(i)
}
// 标识生成器
type generator struct {
maximalSequence int64 // 序列段的最大值
maxinalMachineCode int64 // 机器码的最大值
sequenceLength int64 // 序列段长度
machineCodeLength int64 // 机器码长度
lastTimestamp int64 // 最后回绕时间
getTimestampFunc func() int64 // 获取时间段的方法
}
func (g generator) MaxinalMachineCode() int64 {
return g.maxinalMachineCode
}
func (g generator) MaximalSequence() int64 {
return g.maximalSequence
}
func (g *generator) getTimestamp(reback bool) int64 {
newTimestamp := g.getTimestampFunc()
for reback && g.lastTimestamp == newTimestamp {
time.Sleep(time.Microsecond * 10)
newTimestamp = g.getTimestampFunc()
}
g.lastTimestamp = newTimestamp
return newTimestamp
}
type binaryFormater struct {
generator
}
func newBinaryFormatter(start int64, sequenceLength, machineCodeLength int64) (*binaryFormater, error) {
return &binaryFormater{
generator: generator{
maximalSequence: int64(-1 ^ (-1 << sequenceLength)),
maxinalMachineCode: int64(-1 ^ (-1 << machineCodeLength)),
sequenceLength: sequenceLength,
machineCodeLength: machineCodeLength,
getTimestampFunc: func() int64 { return time.Now().Unix() - start },
},
}, nil
}
func (f *binaryFormater) Format(machine, serial int64, reback bool) Id {
timestamp := f.getTimestamp(reback)
return Id(timestamp<<(f.sequenceLength+f.machineCodeLength) | machine<<f.sequenceLength | serial)
}
type decimalFormater struct {
generator
supportReback bool
}
const (
decimalMaxLength = 19
)
func newDecimalFormater(format string, serialLength, machineLength int) (*decimalFormater, error) {
if len(format)+serialLength+machineLength > decimalMaxLength {
return nil, errors.New("the data length is out of limit")
}
serialShift := int64(math.Pow10(serialLength))
machineShift := int64(math.Pow10(machineLength))
return &decimalFormater{
generator: generator{
sequenceLength: serialShift,
maximalSequence: serialShift - 1,
machineCodeLength: machineShift,
maxinalMachineCode: machineShift - 1,
getTimestampFunc: func() int64 {
now := time.Now()
v := now.Format(format)
r, _ := strconv.ParseInt(v, 10, 64)
return r
},
},
supportReback: len(format) == 14,
}, nil
}
func (f *decimalFormater) Format(machine, serial int64, reback bool) Id {
timestamp := f.getTimestamp(reback)
return Id(timestamp*f.sequenceLength*f.machineCodeLength + machine*f.sequenceLength + serial)
}

34
idGenerator/formater_test.go Normal file
View File

@ -0,0 +1,34 @@
package idgenerator
import (
"testing"
)
func TestBinary(t *testing.T) {
f, _ := newBinaryFormatter(DefaultStartTimeStamp, 16, 12)
t.Log(f.maxinalMachineCode, f.maximalSequence)
}
func TestDecimal(t *testing.T) {
f, _ := newDecimalFormater(YYYYMMDDHHmmss, 4, 1)
t.Log(f.maxinalMachineCode, f.maxinalMachineCode)
t.Log(f.Format(22333, 9, false))
}
func TestDecimalMonth111(t *testing.T) {
f, _ := newDecimalFormater(YYYYMMDD, 4, 1)
t.Log(f.maxinalMachineCode, f.maxinalMachineCode)
t.Log(f.Format(233, 9, false))
}
func TestBinaryTimestamp(t *testing.T) {
f, _ := newBinaryFormatter(DefaultStartTimeStamp, 10, 4)
for i := 0; i < 100; i++ {
if i%7 == 0 {
t.Log(f.Format(int64(i), 0xF, true))
} else {
t.Log(f.Format(int64(i), 0xF, false))
}
}
}

124
idGenerator/generator.go Normal file
View File

@ -0,0 +1,124 @@
package idgenerator
import (
"time"
"github.com/charlienet/go-mixed/idGenerator/store"
"github.com/charlienet/go-mixed/mathx"
"github.com/charlienet/go-mixed/redis"
)
const (
defaultDoubleBufferStep int64 = 50
)
var DefaultStartTimeStamp = time.Date(2023, 1, 1, 0, 0, 0, 0, time.Local).Unix()
type opt func(*idGenerator) error
type Option struct {
TimeFormat string
SerialLength int
MachineLength int
}
type idGenerator struct {
store storage // 外部存储
formater formater // 格式化器
buffer *doubleBuffer // 序列缓冲
}
func WithMem(machineCode int64) opt {
return WithStore(store.NewMemStore(machineCode))
}
func WithRedis(key string, rdb redis.Client) opt {
return WithStore(store.NewRedisStore(key, rdb))
}
func WithStore(s storage) opt {
return func(ig *idGenerator) error {
ig.store = s
return nil
}
}
func WithDecimalFormater(format string, serialLength, machineLength int) opt {
return func(ig *idGenerator) error {
f, err := newDecimalFormater(format, serialLength, machineLength)
if err != nil {
return err
}
ig.formater = f
return nil
}
}
func WithBinaryFormatter(start int64, serialLength, machineLength int64) opt {
return func(ig *idGenerator) error {
f, err := newBinaryFormatter(start, serialLength, machineLength)
if err != nil {
return err
}
ig.formater = f
return nil
}
}
func New(opts ...opt) (*idGenerator, error) {
g := &idGenerator{}
for _, o := range opts {
err := o(g)
if err != nil {
return nil, err
}
}
if g.store == nil {
g.store = store.NewMemStore(0)
}
_, err := g.store.UpdateMachineCode(g.formater.MaxinalMachineCode()) // 初始化机器码
if err != nil {
return nil, err
}
g.buffer = newDoubleBuffer(g.obtain) // 初始化序列缓冲
return g, nil
}
func (g *idGenerator) WithRedis(key string, rdb redis.Client) *idGenerator {
return g.WithStore(store.NewRedisStore(key, rdb))
}
func (g *idGenerator) WithStore(s storage) *idGenerator {
g.store = s
return g
}
func (g *idGenerator) Next() Id {
serial, reback := g.buffer.allot()
id := g.formater.Format(g.store.MachineCode(), serial, reback)
return id
}
func (g *idGenerator) Close() {
if g.store != nil {
g.store.Close()
}
}
func (g *idGenerator) obtain() (*store.Segment, error) {
step := mathx.Min(defaultDoubleBufferStep, g.formater.MaximalSequence())
s, err := g.store.Assign(0, g.formater.MaximalSequence(), step)
if err != nil {
println("分配失败", err.Error())
}
return s, err
}

151
idGenerator/genterator_test.go Normal file
View File

@ -0,0 +1,151 @@
package idgenerator_test
import (
"sync"
"testing"
idgenerator "github.com/charlienet/go-mixed/idGenerator"
"github.com/charlienet/go-mixed/redis"
"github.com/charlienet/go-mixed/sets"
"github.com/charlienet/go-mixed/tests"
)
var redisOption = redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"}
func TestGenerator(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
generator, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 1, 1),
idgenerator.WithRedis("idgen_test", rdb))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 20; i++ {
t.Log(generator.Next())
}
})
}
func TestDecimalGenerator(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
generator, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 3, 1),
idgenerator.WithRedis("idgen_test", rdb))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 200; i++ {
t.Log(generator.Next())
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestDecimalMonth(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
generator, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDD, 2, 1),
idgenerator.WithRedis("idgen_test", rdb))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 105; i++ {
t.Log(generator.Next())
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestParallelCreate(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
g1, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 3, 1),
idgenerator.WithRedis("idgen_testcccc", rdb))
if err != nil {
panic(err)
}
_ = g1.Next().Int64()
}()
go func() {
defer wg.Done()
g2, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 3, 1),
idgenerator.WithRedis("idgen_testcccc", rdb))
if err != nil {
panic(err)
}
_ = g2.Next().Int64()
}()
wg.Wait()
}, redisOption)
}
func TestParallel(t *testing.T) {
set := sets.NewHashSet[int64]().Sync()
opt := redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"}
_ = set
f := func() {
tests.RunOnRedis(t, func(rdb redis.Client) {
generator, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 3, 1),
idgenerator.WithRedis("idgen_testcccc", rdb))
if err != nil {
t.Fatal(err)
}
defer generator.Close()
generator.Next()
for i := 0; i < 50000; i++ {
id := generator.Next().Int64()
if set.Contains(id) {
panic("生成重复")
}
set.Add(id)
}
}, opt)
}
var wg sync.WaitGroup
for i := 0; i < 6; i++ {
wg.Add(1)
go func() {
defer wg.Done()
f()
}()
}
wg.Wait()
}
func BenchmarkGenerator(b *testing.B) {
tests.RunOnRedis(b, func(rdb redis.Client) {
b.Run("bbb", func(b *testing.B) {
generator, err := idgenerator.New(
idgenerator.WithDecimalFormater(idgenerator.YYYYMMDDHHmmss, 3, 1),
idgenerator.WithRedis("idgen_test", rdb))
if err != nil {
b.Fatal(err)
}
for i := 0; i < 999; i++ {
generator.Next()
}
})
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}

118
idGenerator/id_generator.go
View File

@ -1,118 +0,0 @@
package idgenerator
import (
"fmt"
"math"
"time"
_ "unsafe"
)
// 时间段开始时间 2022-01-01
const startTimeStamp = 1640966400
const (
MachineIdBits = uint(8) //机器id所占的位数
SequenceBits = uint(12) //序列所占的位数
MachineIdMax = int64(-1 ^ (-1 << MachineIdBits)) //支持的最大机器id数量
SequenceMask = uint64(-1 ^ (-1 << SequenceBits)) //
MachineIdShift = uint64(SequenceBits) //机器id左移位数
TimestampShift = uint64(SequenceBits + MachineIdBits) //时间戳左移位数
)
type TimePrecision int
const (
Second TimePrecision = iota // 秒
Minute // 分
Day // 日
)
type Config struct {
Machine int
TimeScope TimePrecision
}
type Generator struct {
machine uint64
timeScope TimePrecision // 时间段精度
timestamp uint64 // 上次生成时间
sequence uint64 // 上次使用序列
}
type Id struct {
machine uint64 // 机器标识
Scope int // 时间精度
Timestamp uint64 // 生成时间
Sequence uint64 // 标识序列
}
func New(cfg Config) *Generator {
return &Generator{
machine: uint64(cfg.Machine),
timeScope: cfg.TimeScope,
}
}
func (g *Generator) Next() Id {
now := currentTimestamp()
if g.timestamp == now && g.sequence == 0 {
fmt.Println(time.Now().Format("2006-01-02 15:04:05.000"), "下一个时间点")
for now <= g.timestamp {
// runtime.Gosched()
now = currentTimestamp()
}
}
g.timestamp = now // 标识生成时间
g.sequence = (g.sequence + 1) & SequenceMask // 生成下一序列，超过最大值时返回零
return Id{
machine: g.machine,
Timestamp: g.timestamp,
Sequence: g.sequence,
}
}
func (i Id) Id() uint64 {
return i.Timestamp<<TimestampShift | i.machine<<MachineIdShift | i.Sequence
}
func (i Id) genCheck() uint64 {
return math.MaxUint64
}
func (i Id) String() string {
return fmt.Sprintf("Time:%d scope: %d Machine: %d Ser:%06d id:%d", i.Timestamp, i.Scope, i.machine, i.Sequence, i.Id())
}
func (g *Generator) NextId() uint64 {
return g.Next().Id()
}
func (g *Generator) Batch(num int) []uint64 {
ret := make([]uint64, num)
for i := 0; i < num; i++ {
ret[i] = g.NextId()
}
return ret
}
func Decode(id uint64) Id {
return Id{}
}
func Verify(id uint64) bool {
return false
}
func currentTimestamp() uint64 {
return uint64(time.Now().Unix() - startTimeStamp)
}
//go:linkname runtimeNano runtime.nanotime
func runtimeNano() int64

32
idGenerator/id_generator_test.go
View File

@ -1,32 +0,0 @@
package idgenerator_test
import (
"testing"
idgenerator "github.com/charlienet/go-mixed/idGenerator"
)
func TestNextId(t *testing.T) {
generator := idgenerator.New(idgenerator.Config{})
for i := 0; i < 10; i++ {
t.Log(generator.Next().Id())
}
}
func TestBatch(t *testing.T) {
generator := idgenerator.New(idgenerator.Config{})
b := generator.Batch(20)
for _, i := range b {
t.Log(i)
}
}
func TestNext(t *testing.T) {
generator := idgenerator.New(idgenerator.Config{})
for i := 0; i < 10; i++ {
t.Log(generator.Next())
}
}

114
idGenerator/readme.md Normal file
View File

@ -0,0 +1,114 @@
id生成服务
标识生成服务，可用于分布式环境的标识生成服务，本服务可以多实例部署。
要求
1. 全局唯一: 必须保证生成的 ID 是全局性唯一的；
2. 趋势递增: 生成的 ID 需要按照某种规则有序，便于数据库的写入和排序操作；
3. 单调递增: 后生成的标识大于前面生成的标识
3. 可用性: 需要保证高并发下的可用性。
4. 自主性: 分布式环境下不依赖中心认证即可自行生成 ID；
5. 安全性: 不暴露系统和业务的信息。在一些业务场景下，需要 ID 无规则或者不规则;
标识组成
1. 时间信息，从当前时间中根据模板获取数据段。
号段模式
![号段模式](assets/双缓存号段分配.webp)
批量获取标识，业务应用使用后临近剩余数量时生成新的号段，使用双标识缓冲区方式保存。在活动缓冲区使用量达到限定值时在备缓冲区放置新的号段。
号段分配器
数据结构
1. 最大序列
2. 步长
3. 版本号
分配步骤
1. 服务端按照KEY存储当前分配的最大值
2. 获取是
序列回绕，当序列递增到最大值时恢复为起始值重新开始。在使用中按照业务场景同一时间段中不能耗尽序列值。当序列发生回绕时分配器返回已回绕（backward）。
操作接口
1. 客户端注册标识，客户端使用应用名、规则(前缀、步长、阈值、初始值)进行注册。注册成功后返回标识
标识生成规则，时间段、数字序列
1. 前缀: 使用模板方式生成，生成参数包括当前时间，时间戳。
2. 位数: 表示数据序列包含的位数，如4位表示序列
3. 初始值：序列初始值，序列到达最大值时返回到初始值。
标识生成统一包含时间信息，由于在分布式环境中各应用节点时间可能并不同步，可能造成应用节点生成重复。
1. 应用端获取片段时记录当前时间(T1)
2. 请求服务端分配片段
3. 服务端记录报文到达时间(T2)
4. 服务端处理完成后记录离开时间(T3)
5. 应用端接收到响应时，记录时间(T4)
时间周期延时
1. Delay = ( T4 - T1 ) - ( T3 - T2 )
2. Offset = (( T2- T1 ) + ( T3 – T4 )) / 2
应用端使用延时和偏移计算当前时间
标识服务服务器多活
微信序列号生成方案
微信序列号跟用户 uin 绑定，具有以下性质：递增的 64 位整形；使用每个用户独立的 64 位 sequence 的体系，而不是用一个全局的 64 位（或更高位） sequence ，很大原因是全局唯一的 sequence 会有非常严重的申请互斥问题，不容易去实现一个高性能高可靠的架构。其实现方式包含如下两个关键点：
1）步进式持久化：增加一个缓存中间层，内存中缓存最近一个分配出现的 sequence：cur_seq，以及分配上限：max_seq；分配 sequence 时，将 cur_seq++，与分配上限 max_seq 比较，如果 cur_seq > max_seq，将分配上限提升一个步长 max_seq += step，并持久化 max_seq；重启时，读出持久化的 max_seq，赋值给 cur_seq。此种处理方式可以降低持久化的硬盘 IO 次数，可以系统的整体吞吐量。
2）分号段共享存储：引入号段 section 的概念，uin 相邻的一段用户属于一个号段，共享一个 max_seq。该处理方式可以大幅减少 max_seq 数据的大小，同时可以进一步地降低 IO 次数。
二进制模式
按照二进制模式处理序列号生成，各数据段按照各自的位长度组装
十进制模式
缺陷
当发生序列回绕时，不同的时间序列有不同的处理。如时间序列精度为日，那么在发生回绕时认为此时的回绕不能发生。如果应用在当日内被重启，并且发生了回绕，当前机制无法检测此问题。
序列记录可以依赖于外部存储，如Redis，DB等。当外部存储失效时序列可能会发生冲突。此时当时间段没有前进时可能发生重复。一般认为DB为不失效存储。
需要烦精确的选择时间段和序列的相对关系，一般情况下
ID分配器工作流程
初始化
1. 创建生成器
2. 创建外部存储
3. 指定生成器生成规则，机器码长度，序列长度，序列化格式。
4. 创建双缓存，使用外部存储分配数据段。
标识分配
1. 向序列缓冲器获取下一序列
2. 向时间段分配器获取下一时间戳（传入是否回旋）
4. 由标识组装器获取
格式化器
十进制格式化，时间段格式串
二进制格式化，起始时间，时间精度

11
idGenerator/store.go Normal file
View File

@ -0,0 +1,11 @@
package idgenerator
import "github.com/charlienet/go-mixed/idGenerator/store"
// 序列存储分配器
type storage interface {
MachineCode() int64 // 当前机器码
UpdateMachineCode(max int64) (int64, error) // 更新机器标识
Assign(min, max, step int64) (*store.Segment, error) // 分配号段
Close()
}

60
idGenerator/store/mem_store.go Normal file
View File

@ -0,0 +1,60 @@
package store
import (
"sync"
"github.com/charlienet/go-mixed/mathx"
)
type memStore struct {
mu sync.Mutex
machine int64
current int64
}
func NewMemStore(machineCode int64) *memStore {
return &memStore{machine: machineCode}
}
func (s *memStore) UpdateMachineCode(max int64) (int64, error) {
return s.machine, nil
}
func (s *memStore) MachineCode() int64 {
return s.machine
}
func (s *memStore) Assign(min, max, step int64) (*Segment, error) {
s.mu.Lock()
defer s.mu.Unlock()
step = mathx.Min(step, max)
start := mathx.Max(s.current, min)
end := start + step
reback := false
if start >= max {
start = min
end = step
s.current = end
reback = true
}
if end > max {
end = max
s.current = end
}
s.current = end
return &Segment{
start: start,
current: start,
end: end,
reback: reback,
}, nil
}
func (s *memStore) Close() {
}

21
idGenerator/store/mem_store_test.go Normal file
View File

@ -0,0 +1,21 @@
package store_test
import (
"testing"
"github.com/charlienet/go-mixed/idGenerator/store"
)
func TestMemSmall(t *testing.T) {
s := store.NewMemStore(2)
for i := 0; i < 10; i++ {
t.Log(s.Assign(1, 9, 20))
}
}
func TestMemBig(t *testing.T) {
s := store.NewMemStore(2)
for i := 0; i < 10; i++ {
t.Log(s.Assign(0, 99, 18))
}
}

2
idGenerator/store/mysql_store.go Normal file
View File

@ -0,0 +1,2 @@
package store

194
idGenerator/store/redis_store.go Normal file
View File

@ -0,0 +1,194 @@
package store
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/charlienet/go-mixed/rand"
"github.com/charlienet/go-mixed/redis"
)
const (
allocatedKey = "allocated"
sequenceKey = "sequence"
delaySecond = 10
)
const (
// 机器码分配及保活，检查机器码键值是否匹配，如匹配成功对键进行延时。如不匹配需要重新申请机器码
// 请求参数 机器码，机器识别码，机器码最大值
machineLua = `
local newKey = KEYS[1]..":"..ARGV[1]
if redis.call("GET", newKey) == ARGV[2] then
redis.call("EXPIRE", newKey, 10)
return tonumber(ARGV[1])
else
for i = 0, ARGV[3], 1 do
newKey = KEYS[1]..":"..tostring(i)
if redis.call("EXISTS", newKey) == 0 then
redis.call("SET", newKey, ARGV[2], "EX", 10)
return i
end
end
return -1
end`
// 序列分配min, max, step
segmentLua = `
local key = KEYS[1]
local min = tonumber(ARGV[1])
local max = tonumber(ARGV[2])
local step = tonumber(ARGV[3])
if step > max then
step = max
end
if redis.call("EXISTS", key) == 0 then
redis.call("SET", key, step)
return {min, step, 0}
end
local begin = tonumber(redis.call("GET", key))
local increase = redis.call("INCRBY", key, step)
local reback = 0
if begin >= max then
begin = min
increase = step
redis.call("SET", key, step)
reback = 1
end
if increase > max then
increase = max
redis.call("SET", key, increase)
end
return {begin, increase, reback}
`
)
type redisStore struct {
rdb redis.Client
machinekey string // 机器码键
sequenceKey string // 序列键
value string // 随机键值
machineCode int64 // 机器码
close chan struct{} // 关闭保活协程
isRunning bool // 是否已经关闭
mu sync.Mutex
}
func NewRedisStore(key string, rdb redis.Client) *redisStore {
return &redisStore{
rdb: rdb,
machinekey: rdb.JoinKeys(key, allocatedKey),
sequenceKey: rdb.JoinKeys(key, sequenceKey),
value: rand.Hex.Generate(24),
close: make(chan struct{}),
}
}
// 分配机器标识，分配值为-1时表示分配失败
func (s *redisStore) UpdateMachineCode(max int64) (int64, error) {
err := s.updateMachine(max)
if err != nil {
return -1, err
}
// 关闭原协程，开启新的保活协程
// if s.isRunning {
// s.close <- struct{}{}
// }
// if !s.isRunning {
// s.close <- struct{}{}
go s.keepAlive(max)
// }
return s.machineCode, nil
}
func (s *redisStore) MachineCode() int64 {
return s.machineCode
}
func (s *redisStore) Assign(min, max, step int64) (*Segment, error) {
s.mu.Lock()
defer s.mu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
defer cancel()
// 序列段分配
key := s.rdb.JoinKeys(s.sequenceKey, fmt.Sprintf("%v", s.machineCode))
r, err := s.rdb.Eval(ctx, segmentLua, []string{key}, min, max, step).Result()
if err != nil {
return &Segment{}, err
}
start, end, reback := split(r)
return &Segment{start: start, end: end, current: start, reback: reback}, err
}
func split(r any) (start, end int64, reback bool) {
if result, ok := r.([]any); ok {
start = result[0].(int64)
end = result[1].(int64)
reback = result[2].(int64) == 1
}
return
}
func (s *redisStore) updateMachine(max int64) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
r, err := s.rdb.Eval(ctx, machineLua, []string{s.machinekey}, s.machineCode, s.value, max).Result()
if err != nil {
return err
}
if r == nil {
return errors.New("failed to obtain machine code")
}
s.machineCode = r.(int64)
if s.machineCode == -1 {
return errors.New("machine code allocation failed")
}
return nil
}
func (s *redisStore) Close() {
s.close <- struct{}{}
s.isRunning = false
}
func (s *redisStore) keepAlive(max int64) {
t := time.NewTicker(time.Second * (delaySecond / 3))
defer t.Stop()
for {
select {
case <-t.C:
// println("当前机器码:", s.machineCode)
err := s.updateMachine(max)
if err != nil {
fmt.Println("err:", err.Error())
}
case <-s.close:
println("保活停止")
return
}
}
}

106
idGenerator/store/redis_store_test.go Normal file
View File

@ -0,0 +1,106 @@
package store_test
import (
"testing"
"time"
"github.com/charlienet/go-mixed/idGenerator/store"
"github.com/charlienet/go-mixed/redis"
"github.com/charlienet/go-mixed/tests"
)
func TestSmallSerail(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("sss", rdb)
for i := 0; i < 5; i++ {
t.Log(s.Assign(0, 9, 20))
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestSmallAssign(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("sss", rdb)
for i := 0; i < 10; i++ {
t.Log(s.Assign(0, 9, 30))
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestBigAssign(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("sss", rdb)
for i := 0; i < 102; i++ {
t.Log(s.Assign(0, 99, 10))
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestRedisAssign(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("sss", rdb)
for i := 0; i < 10; i++ {
t.Log(s.Assign(21, 99, 30))
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestFullRedisAssign(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("sss", rdb)
for i := 0; i < 10; i++ {
t.Log(s.Assign(0, 999, 99))
}
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456"})
}
func TestUpdateMachineCode(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
for i := 0; i < 20; i++ {
s := store.NewRedisStore("id", rdb)
code, err := s.UpdateMachineCode(99)
t.Log("获取到机器标识:", code, err)
if err != nil {
return
}
time.Sleep(time.Millisecond * 100)
// s.Close()
}
time.Sleep(time.Second * 10)
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456", Prefix: "cacc"})
}
func TestUpdate(t *testing.T) {
tests.RunOnRedis(t, func(rdb redis.Client) {
s := store.NewRedisStore("id", rdb)
s.UpdateMachineCode(99)
t.Log(s.MachineCode())
s.UpdateMachineCode(99)
t.Log(s.MachineCode())
s2 := store.NewRedisStore("id", rdb)
s2.UpdateMachineCode(99)
t.Log(s2.MachineCode())
}, redis.ReidsOption{Addr: "192.168.123.50:6379", Password: "123456", Prefix: "cacc"})
}

33
idGenerator/store/segment.go Normal file
View File

@ -0,0 +1,33 @@
package store
import "fmt"
// 号段
type Segment struct {
start int64
end int64
current int64
reback bool
}
func (s *Segment) Allot() int64 {
s.current++
return s.current
}
func (s *Segment) IsEnding() bool {
return (s.current - s.start) > (s.end - s.current)
}
func (s *Segment) IsEmpty() bool {
return s.current == s.end
}
func (s *Segment) Reback() bool {
// println("回旋确认:", s.reback, s.current == (s.start+1))
return s.reback && s.current == (s.start+1)
}
func (s *Segment) String() string {
return fmt.Sprintf("start:%d-%d(%v)", s.start, s.end, s.reback)
}