@@ -27,8 +27,8 @@ import (
27
27
type (
28
28
// Node is a pool of workers.
29
29
Node struct {
30
- Name string
31
30
NodeID string
31
+ PoolName string
32
32
poolStream * streaming.Stream // pool event stream for dispatching jobs
33
33
poolSink * streaming.Sink // pool event sink
34
34
nodeStream * streaming.Stream // node event stream for receiving worker events
@@ -93,14 +93,14 @@ const (
93
93
// The options WithClientOnly can be used to create a node that can only be used
94
94
// to dispatch jobs. Such a node does not route or process jobs in the
95
95
// background.
96
- func AddNode (ctx context.Context , name string , rdb * redis.Client , opts ... NodeOption ) (* Node , error ) {
96
+ func AddNode (ctx context.Context , poolName string , rdb * redis.Client , opts ... NodeOption ) (* Node , error ) {
97
97
o := parseOptions (opts ... )
98
98
logger := o .logger
99
99
nodeID := ulid .Make ().String ()
100
100
if logger == nil {
101
101
logger = pulse .NoopLogger ()
102
102
} else {
103
- logger = logger .WithPrefix ("pool" , name , "node" , nodeID )
103
+ logger = logger .WithPrefix ("pool" , poolName , "node" , nodeID )
104
104
}
105
105
logger .Info ("options" ,
106
106
"client_only" , o .clientOnly ,
@@ -110,18 +110,18 @@ func AddNode(ctx context.Context, name string, rdb *redis.Client, opts ...NodeOp
110
110
"pending_job_ttl" , o .pendingJobTTL ,
111
111
"job_sink_block_duration" , o .jobSinkBlockDuration ,
112
112
"ack_grace_period" , o .ackGracePeriod )
113
- wsm , err := rmap .Join (ctx , shutdownMapName (name ), rdb , rmap .WithLogger (logger ))
113
+ wsm , err := rmap .Join (ctx , shutdownMapName (poolName ), rdb , rmap .WithLogger (logger ))
114
114
if err != nil {
115
- return nil , fmt .Errorf ("AddNode: failed to join shutdown replicated map %q: %w" , shutdownMapName (name ), err )
115
+ return nil , fmt .Errorf ("AddNode: failed to join shutdown replicated map %q: %w" , shutdownMapName (poolName ), err )
116
116
}
117
117
if wsm .Len () > 0 {
118
- return nil , fmt .Errorf ("AddNode: pool %q is shutting down" , name )
118
+ return nil , fmt .Errorf ("AddNode: pool %q is shutting down" , poolName )
119
119
}
120
- poolStream , err := streaming .NewStream (poolStreamName (name ), rdb ,
120
+ poolStream , err := streaming .NewStream (poolStreamName (poolName ), rdb ,
121
121
soptions .WithStreamMaxLen (o .maxQueuedJobs ),
122
122
soptions .WithStreamLogger (logger ))
123
123
if err != nil {
124
- return nil , fmt .Errorf ("AddNode: failed to create pool job stream %q: %w" , poolStreamName (name ), err )
124
+ return nil , fmt .Errorf ("AddNode: failed to create pool job stream %q: %w" , poolStreamName (poolName ), err )
125
125
}
126
126
var (
127
127
wm * rmap.Map
@@ -134,47 +134,47 @@ func AddNode(ctx context.Context, name string, rdb *redis.Client, opts ...NodeOp
134
134
nodeReader * streaming.Reader
135
135
)
136
136
if ! o .clientOnly {
137
- wm , err = rmap .Join (ctx , workerMapName (name ), rdb , rmap .WithLogger (logger ))
137
+ wm , err = rmap .Join (ctx , workerMapName (poolName ), rdb , rmap .WithLogger (logger ))
138
138
if err != nil {
139
- return nil , fmt .Errorf ("AddNode: failed to join pool workers replicated map %q: %w" , workerMapName (name ), err )
139
+ return nil , fmt .Errorf ("AddNode: failed to join pool workers replicated map %q: %w" , workerMapName (poolName ), err )
140
140
}
141
141
workerIDs := wm .Keys ()
142
142
logger .Info ("joined" , "workers" , workerIDs )
143
- jm , err = rmap .Join (ctx , jobsMapName (name ), rdb , rmap .WithLogger (logger ))
143
+ jm , err = rmap .Join (ctx , jobsMapName (poolName ), rdb , rmap .WithLogger (logger ))
144
144
if err != nil {
145
- return nil , fmt .Errorf ("AddNode: failed to join pool jobs replicated map %q: %w" , jobsMapName (name ), err )
145
+ return nil , fmt .Errorf ("AddNode: failed to join pool jobs replicated map %q: %w" , jobsMapName (poolName ), err )
146
146
}
147
- jpm , err = rmap .Join (ctx , jobPayloadsMapName (name ), rdb , rmap .WithLogger (logger ))
147
+ jpm , err = rmap .Join (ctx , jobPayloadsMapName (poolName ), rdb , rmap .WithLogger (logger ))
148
148
if err != nil {
149
- return nil , fmt .Errorf ("AddNode: failed to join pool job payloads replicated map %q: %w" , jobPayloadsMapName (name ), err )
149
+ return nil , fmt .Errorf ("AddNode: failed to join pool job payloads replicated map %q: %w" , jobPayloadsMapName (poolName ), err )
150
150
}
151
- km , err = rmap .Join (ctx , keepAliveMapName (name ), rdb , rmap .WithLogger (logger ))
151
+ km , err = rmap .Join (ctx , keepAliveMapName (poolName ), rdb , rmap .WithLogger (logger ))
152
152
if err != nil {
153
- return nil , fmt .Errorf ("AddNode: failed to join pool keep-alive replicated map %q: %w" , keepAliveMapName (name ), err )
153
+ return nil , fmt .Errorf ("AddNode: failed to join pool keep-alive replicated map %q: %w" , keepAliveMapName (poolName ), err )
154
154
}
155
- tm , err = rmap .Join (ctx , tickerMapName (name ), rdb , rmap .WithLogger (logger ))
155
+ tm , err = rmap .Join (ctx , tickerMapName (poolName ), rdb , rmap .WithLogger (logger ))
156
156
if err != nil {
157
- return nil , fmt .Errorf ("AddNode: failed to join pool ticker replicated map %q: %w" , tickerMapName (name ), err )
157
+ return nil , fmt .Errorf ("AddNode: failed to join pool ticker replicated map %q: %w" , tickerMapName (poolName ), err )
158
158
}
159
159
poolSink , err = poolStream .NewSink (ctx , "events" ,
160
160
soptions .WithSinkBlockDuration (o .jobSinkBlockDuration ),
161
161
soptions .WithSinkAckGracePeriod (o .ackGracePeriod ))
162
162
if err != nil {
163
- return nil , fmt .Errorf ("AddNode: failed to create events sink for stream %q: %w" , poolStreamName (name ), err )
163
+ return nil , fmt .Errorf ("AddNode: failed to create events sink for stream %q: %w" , poolStreamName (poolName ), err )
164
164
}
165
165
}
166
- nodeStream , err = streaming .NewStream (nodeStreamName (name , nodeID ), rdb , soptions .WithStreamLogger (logger ))
166
+ nodeStream , err = streaming .NewStream (nodeStreamName (poolName , nodeID ), rdb , soptions .WithStreamLogger (logger ))
167
167
if err != nil {
168
- return nil , fmt .Errorf ("AddNode: failed to create node event stream %q: %w" , nodeStreamName (name , nodeID ), err )
168
+ return nil , fmt .Errorf ("AddNode: failed to create node event stream %q: %w" , nodeStreamName (poolName , nodeID ), err )
169
169
}
170
170
nodeReader , err = nodeStream .NewReader (ctx , soptions .WithReaderBlockDuration (o .jobSinkBlockDuration ), soptions .WithReaderStartAtOldest ())
171
171
if err != nil {
172
- return nil , fmt .Errorf ("AddNode: failed to create node event reader for stream %q: %w" , nodeStreamName (name , nodeID ), err )
172
+ return nil , fmt .Errorf ("AddNode: failed to create node event reader for stream %q: %w" , nodeStreamName (poolName , nodeID ), err )
173
173
}
174
174
175
175
p := & Node {
176
- Name : name ,
177
176
NodeID : nodeID ,
177
+ PoolName : poolName ,
178
178
keepAliveMap : km ,
179
179
workerMap : wm ,
180
180
jobsMap : jm ,
@@ -225,10 +225,10 @@ func (node *Node) AddWorker(ctx context.Context, handler JobHandler) (*Worker, e
225
225
node .lock .Lock ()
226
226
defer node .lock .Unlock ()
227
227
if node .closing {
228
- return nil , fmt .Errorf ("AddWorker: pool %q is closed" , node .Name )
228
+ return nil , fmt .Errorf ("AddWorker: pool %q is closed" , node .PoolName )
229
229
}
230
230
if node .clientOnly {
231
- return nil , fmt .Errorf ("AddWorker: pool %q is client-only" , node .Name )
231
+ return nil , fmt .Errorf ("AddWorker: pool %q is client-only" , node .PoolName )
232
232
}
233
233
w , err := newWorker (ctx , node , handler )
234
234
if err != nil {
@@ -303,7 +303,7 @@ func (node *Node) DispatchJob(ctx context.Context, key string, payload []byte) e
303
303
node .lock .Lock ()
304
304
if node .closing {
305
305
node .lock .Unlock ()
306
- return fmt .Errorf ("DispatchJob: pool %q is closed" , node .Name )
306
+ return fmt .Errorf ("DispatchJob: pool %q is closed" , node .PoolName )
307
307
}
308
308
job := marshalJob (& Job {Key : key , Payload : payload , CreatedAt : time .Now (), NodeID : node .NodeID })
309
309
eventID , err := node .poolStream .Add (ctx , evStartJob , job )
@@ -344,7 +344,7 @@ func (node *Node) StopJob(ctx context.Context, key string) error {
344
344
node .lock .Lock ()
345
345
defer node .lock .Unlock ()
346
346
if node .closing {
347
- return fmt .Errorf ("StopJob: pool %q is closed" , node .Name )
347
+ return fmt .Errorf ("StopJob: pool %q is closed" , node .PoolName )
348
348
}
349
349
if _ , err := node .poolStream .Add (ctx , evStopJob , marshalJobKey (key )); err != nil {
350
350
return fmt .Errorf ("StopJob: failed to add stop job to stream %q: %w" , node .poolStream .Name , err )
@@ -383,7 +383,7 @@ func (node *Node) NotifyWorker(ctx context.Context, key string, payload []byte)
383
383
node .lock .Lock ()
384
384
defer node .lock .Unlock ()
385
385
if node .closing {
386
- return fmt .Errorf ("NotifyWorker: pool %q is closed" , node .Name )
386
+ return fmt .Errorf ("NotifyWorker: pool %q is closed" , node .PoolName )
387
387
}
388
388
if _ , err := node .poolStream .Add (ctx , evNotify , marshalNotification (key , payload )); err != nil {
389
389
return fmt .Errorf ("NotifyWorker: failed to add notification to stream %q: %w" , node .poolStream .Name , err )
@@ -421,7 +421,7 @@ func (node *Node) Shutdown(ctx context.Context) error {
421
421
}
422
422
423
423
// Now clean up the shutdown replicated map.
424
- wsm , err := rmap .Join (ctx , shutdownMapName (node .Name ), node .rdb , rmap .WithLogger (node .logger ))
424
+ wsm , err := rmap .Join (ctx , shutdownMapName (node .PoolName ), node .rdb , rmap .WithLogger (node .logger ))
425
425
if err != nil {
426
426
node .logger .Error (fmt .Errorf ("Shutdown: failed to join shutdown map for cleanup: %w" , err ))
427
427
}
@@ -535,7 +535,7 @@ func (node *Node) routeWorkerEvent(ctx context.Context, ev *streaming.Event) err
535
535
key := unmarshalJobKey (ev .Payload )
536
536
activeWorkers := node .activeWorkers ()
537
537
if len (activeWorkers ) == 0 {
538
- return fmt .Errorf ("routeWorkerEvent: no active worker in pool %q" , node .Name )
538
+ return fmt .Errorf ("routeWorkerEvent: no active worker in pool %q" , node .PoolName )
539
539
}
540
540
wid := activeWorkers [node .h .Hash (key , int64 (len (activeWorkers )))]
541
541
@@ -606,14 +606,14 @@ func (node *Node) ackWorkerEvent(ctx context.Context, ev *streaming.Event) {
606
606
// dispatched the job.
607
607
if pending .EventName == evStartJob {
608
608
_ , nodeID := unmarshalJobKeyAndNodeID (pending .Payload )
609
- stream , err := streaming .NewStream (nodeStreamName (node .Name , nodeID ), node .rdb , soptions .WithStreamLogger (node .logger ))
609
+ stream , err := streaming .NewStream (nodeStreamName (node .PoolName , nodeID ), node .rdb , soptions .WithStreamLogger (node .logger ))
610
610
if err != nil {
611
- node .logger .Error (fmt .Errorf ("ackWorkerEvent: failed to create node event stream %q: %w" , nodeStreamName (node .Name , nodeID ), err ))
611
+ node .logger .Error (fmt .Errorf ("ackWorkerEvent: failed to create node event stream %q: %w" , nodeStreamName (node .PoolName , nodeID ), err ))
612
612
return
613
613
}
614
614
ack .EventID = pending .ID
615
615
if _ , err := stream .Add (ctx , evDispatchReturn , marshalAck (ack ), soptions .WithOnlyIfStreamExists ()); err != nil {
616
- node .logger .Error (fmt .Errorf ("ackWorkerEvent: failed to dispatch return to stream %q: %w" , nodeStreamName (node .Name , nodeID ), err ))
616
+ node .logger .Error (fmt .Errorf ("ackWorkerEvent: failed to dispatch return to stream %q: %w" , nodeStreamName (node .PoolName , nodeID ), err ))
617
617
}
618
618
}
619
619
0 commit comments