-
Notifications
You must be signed in to change notification settings - Fork 4.2k
/
lifetime_watcher.go
431 lines (366 loc) · 13.3 KB
/
lifetime_watcher.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package api
import (
"errors"
"math/rand"
"strings"
"sync"
"time"
"github.com/cenkalti/backoff/v4"
)
var (
ErrLifetimeWatcherMissingInput = errors.New("missing input")
ErrLifetimeWatcherMissingSecret = errors.New("missing secret")
ErrLifetimeWatcherNotRenewable = errors.New("secret is not renewable")
ErrLifetimeWatcherNoSecretData = errors.New("returned empty secret data")
// Deprecated; kept for compatibility
ErrRenewerMissingInput = errors.New("missing input to renewer")
ErrRenewerMissingSecret = errors.New("missing secret to renew")
ErrRenewerNotRenewable = errors.New("secret is not renewable")
ErrRenewerNoSecretData = errors.New("returned empty secret data")
// DefaultLifetimeWatcherRenewBuffer is the default size of the buffer for renew
// messages on the channel.
DefaultLifetimeWatcherRenewBuffer = 5
// Deprecated: kept for backwards compatibility
DefaultRenewerRenewBuffer = 5
)
//go:generate enumer -type=RenewBehavior -trimprefix=RenewBehavior
type RenewBehavior uint
const (
// RenewBehaviorIgnoreErrors means we will attempt to keep renewing until
// we hit the lifetime threshold. It also ignores errors stemming from
// passing a non-renewable lease in. In practice, this means you simply
// reauthenticate/refetch credentials when the watcher exits. This is the
// default.
RenewBehaviorIgnoreErrors RenewBehavior = iota
// RenewBehaviorRenewDisabled turns off renewal attempts entirely. This
// allows you to simply watch lifetime and have the watcher return at a
// reasonable threshold without actually making Vault calls.
RenewBehaviorRenewDisabled
// RenewBehaviorErrorOnErrors is the "legacy" behavior which always exits
// on some kind of error
RenewBehaviorErrorOnErrors
)
// LifetimeWatcher is a process for watching lifetime of a secret.
//
// watcher, err := client.NewLifetimeWatcher(&LifetimeWatcherInput{
// Secret: mySecret,
// })
// go watcher.Start()
// defer watcher.Stop()
//
// for {
// select {
// case err := <-watcher.DoneCh():
// if err != nil {
// log.Fatal(err)
// }
//
// // Renewal is now over
// case renewal := <-watcher.RenewCh():
// log.Printf("Successfully renewed: %#v", renewal)
// }
// }
//
// `DoneCh` will return if renewal fails, or if the remaining lease duration is
// under a built-in threshold and either renewing is not extending it or
// renewing is disabled. In both cases, the caller should attempt a re-read of
// the secret. Clients should check the return value of the channel to see if
// renewal was successful.
type LifetimeWatcher struct {
l sync.Mutex
client *Client
secret *Secret
grace time.Duration
random *rand.Rand
increment int
doneCh chan error
renewCh chan *RenewOutput
renewBehavior RenewBehavior
stopped bool
stopCh chan struct{}
errLifetimeWatcherNotRenewable error
errLifetimeWatcherNoSecretData error
}
// LifetimeWatcherInput is used as input to the renew function.
type LifetimeWatcherInput struct {
// Secret is the secret to renew
Secret *Secret
// DEPRECATED: this does not do anything.
Grace time.Duration
// Rand is the randomizer to use for underlying randomization. If not
// provided, one will be generated and seeded automatically. If provided, it
// is assumed to have already been seeded.
Rand *rand.Rand
// RenewBuffer is the size of the buffered channel where renew messages are
// dispatched.
RenewBuffer int
// The new TTL, in seconds, that should be set on the lease. The TTL set
// here may or may not be honored by the vault server, based on Vault
// configuration or any associated max TTL values. If specified, the
// minimum of this value and the remaining lease duration will be used
// for grace period calculations.
Increment int
// RenewBehavior controls what happens when a renewal errors or the
// passed-in secret is not renewable.
RenewBehavior RenewBehavior
}
// RenewOutput is the metadata returned to the client (if it's listening) to
// renew messages.
type RenewOutput struct {
// RenewedAt is the timestamp when the renewal took place (UTC).
RenewedAt time.Time
// Secret is the underlying renewal data. It's the same struct as all data
// that is returned from Vault, but since this is renewal data, it will not
// usually include the secret itself.
Secret *Secret
}
// NewLifetimeWatcher creates a new renewer from the given input.
func (c *Client) NewLifetimeWatcher(i *LifetimeWatcherInput) (*LifetimeWatcher, error) {
if i == nil {
return nil, ErrLifetimeWatcherMissingInput
}
secret := i.Secret
if secret == nil {
return nil, ErrLifetimeWatcherMissingSecret
}
random := i.Rand
if random == nil {
// NOTE:
// Rather than a cryptographically secure random number generator (RNG),
// the default behavior uses the math/rand package. The random number is
// used to introduce a slight jitter when calculating the grace period
// for a monitored secret monitoring. This is intended to stagger renewal
// requests to the Vault server, but in a semi-predictable way, so there
// is no need to use a cryptographically secure RNG.
random = rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
}
renewBuffer := i.RenewBuffer
if renewBuffer == 0 {
renewBuffer = DefaultLifetimeWatcherRenewBuffer
}
return &LifetimeWatcher{
client: c,
secret: secret,
increment: i.Increment,
random: random,
doneCh: make(chan error, 1),
renewCh: make(chan *RenewOutput, renewBuffer),
renewBehavior: i.RenewBehavior,
stopped: false,
stopCh: make(chan struct{}),
errLifetimeWatcherNotRenewable: ErrLifetimeWatcherNotRenewable,
errLifetimeWatcherNoSecretData: ErrLifetimeWatcherNoSecretData,
}, nil
}
// Deprecated: exists only for backwards compatibility. Calls
// NewLifetimeWatcher, and sets compatibility flags.
func (c *Client) NewRenewer(i *LifetimeWatcherInput) (*LifetimeWatcher, error) {
if i == nil {
return nil, ErrRenewerMissingInput
}
secret := i.Secret
if secret == nil {
return nil, ErrRenewerMissingSecret
}
renewer, err := c.NewLifetimeWatcher(i)
if err != nil {
return nil, err
}
renewer.renewBehavior = RenewBehaviorErrorOnErrors
renewer.errLifetimeWatcherNotRenewable = ErrRenewerNotRenewable
renewer.errLifetimeWatcherNoSecretData = ErrRenewerNoSecretData
return renewer, err
}
// DoneCh returns the channel where the renewer will publish when renewal stops.
// If there is an error, this will be an error.
func (r *LifetimeWatcher) DoneCh() <-chan error {
return r.doneCh
}
// RenewCh is a channel that receives a message when a successful renewal takes
// place and includes metadata about the renewal.
func (r *LifetimeWatcher) RenewCh() <-chan *RenewOutput {
return r.renewCh
}
// Stop stops the renewer.
func (r *LifetimeWatcher) Stop() {
r.l.Lock()
defer r.l.Unlock()
if !r.stopped {
close(r.stopCh)
r.stopped = true
}
}
// Start starts a background process for watching the lifetime of this secret.
// If renewal is enabled, when the secret has auth data, this attempts to renew
// the auth (token); When the secret has a lease, this attempts to renew the
// lease.
func (r *LifetimeWatcher) Start() {
r.doneCh <- r.doRenew()
}
// Renew is for compatibility with the legacy api.Renewer. Calling Renew
// simply chains to Start.
func (r *LifetimeWatcher) Renew() {
r.Start()
}
type renewFunc func(string, int) (*Secret, error)
// doRenew is a helper for renewing authentication.
func (r *LifetimeWatcher) doRenew() error {
defaultInitialRetryInterval := 10 * time.Second
switch {
case r.secret.Auth != nil:
return r.doRenewWithOptions(true, !r.secret.Auth.Renewable,
r.secret.Auth.LeaseDuration, r.secret.Auth.ClientToken,
r.client.Auth().Token().RenewTokenAsSelf, defaultInitialRetryInterval)
default:
return r.doRenewWithOptions(false, !r.secret.Renewable,
r.secret.LeaseDuration, r.secret.LeaseID,
r.client.Sys().Renew, defaultInitialRetryInterval)
}
}
func (r *LifetimeWatcher) doRenewWithOptions(tokenMode bool, nonRenewable bool, initLeaseDuration int, credString string,
renew renewFunc, initialRetryInterval time.Duration,
) error {
if credString == "" ||
(nonRenewable && r.renewBehavior == RenewBehaviorErrorOnErrors) {
return r.errLifetimeWatcherNotRenewable
}
initialTime := time.Now()
priorDuration := time.Duration(initLeaseDuration) * time.Second
r.calculateGrace(priorDuration, time.Duration(r.increment)*time.Second)
var errorBackoff backoff.BackOff
for {
// Check if we are stopped.
select {
case <-r.stopCh:
return nil
default:
}
var remainingLeaseDuration time.Duration
fallbackLeaseDuration := initialTime.Add(priorDuration).Sub(time.Now())
var renewal *Secret
var err error
switch {
case nonRenewable || r.renewBehavior == RenewBehaviorRenewDisabled:
// Can't or won't renew, just keep the same expiration so we exit
// when it's re-authentication time
remainingLeaseDuration = fallbackLeaseDuration
default:
// Renew the token
renewal, err = renew(credString, r.increment)
if err != nil && strings.Contains(err.Error(), "permission denied") {
// We can't renew since the token doesn't have permission to. Fall back
// to the code path for non-renewable tokens.
nonRenewable = true
continue
}
if err != nil || renewal == nil || (tokenMode && renewal.Auth == nil) {
if r.renewBehavior == RenewBehaviorErrorOnErrors {
if err != nil {
return err
}
if renewal == nil || (tokenMode && renewal.Auth == nil) {
return r.errLifetimeWatcherNoSecretData
}
}
// Calculate remaining duration until initial token lease expires
remainingLeaseDuration = initialTime.Add(time.Duration(initLeaseDuration) * time.Second).Sub(time.Now())
if errorBackoff == nil {
errorBackoff = &backoff.ExponentialBackOff{
MaxElapsedTime: remainingLeaseDuration,
RandomizationFactor: backoff.DefaultRandomizationFactor,
InitialInterval: initialRetryInterval,
MaxInterval: 5 * time.Minute,
Multiplier: 2,
Clock: backoff.SystemClock,
}
errorBackoff.Reset()
}
break
}
errorBackoff = nil
// Push a message that a renewal took place.
select {
case r.renewCh <- &RenewOutput{time.Now().UTC(), renewal}:
default:
}
// Possibly error if we are not renewable
if ((tokenMode && !renewal.Auth.Renewable) || (!tokenMode && !renewal.Renewable)) &&
r.renewBehavior == RenewBehaviorErrorOnErrors {
return r.errLifetimeWatcherNotRenewable
}
// Reset initial time
initialTime = time.Now()
// Grab the lease duration
initLeaseDuration = renewal.LeaseDuration
if tokenMode {
initLeaseDuration = renewal.Auth.LeaseDuration
}
remainingLeaseDuration = time.Duration(initLeaseDuration) * time.Second
}
var sleepDuration time.Duration
if errorBackoff == nil {
sleepDuration = r.calculateSleepDuration(remainingLeaseDuration, priorDuration)
} else {
sleepDuration = errorBackoff.NextBackOff()
if sleepDuration == backoff.Stop {
return err
}
}
// remainingLeaseDuration becomes the priorDuration for the next loop
priorDuration = remainingLeaseDuration
// If we are within grace, return now; or, if the amount of time we
// would sleep would land us in the grace period. This helps with short
// tokens; for example, you don't want a current lease duration of 4
// seconds, a grace period of 3 seconds, and end up sleeping for more
// than three of those seconds and having a very small budget of time
// to renew.
if remainingLeaseDuration <= r.grace || remainingLeaseDuration-sleepDuration <= r.grace {
return nil
}
timer := time.NewTimer(sleepDuration)
select {
case <-r.stopCh:
timer.Stop()
return nil
case <-timer.C:
continue
}
}
}
// calculateSleepDuration calculates the amount of time the LifeTimeWatcher should sleep
// before re-entering its loop.
func (r *LifetimeWatcher) calculateSleepDuration(remainingLeaseDuration, priorDuration time.Duration) time.Duration {
// We keep evaluating a new grace period so long as the lease is
// extending. Once it stops extending, we've hit the max and need to
// rely on the grace duration.
if remainingLeaseDuration > priorDuration {
r.calculateGrace(remainingLeaseDuration, time.Duration(r.increment)*time.Second)
}
// The sleep duration is set to 2/3 of the current lease duration plus
// 1/3 of the current grace period, which adds jitter.
return time.Duration(float64(remainingLeaseDuration.Nanoseconds())*2/3 + float64(r.grace.Nanoseconds())/3)
}
// calculateGrace calculates the grace period based on the minimum of the
// remaining lease duration and the token increment value; it also adds some
// jitter to not have clients be in sync.
func (r *LifetimeWatcher) calculateGrace(leaseDuration, increment time.Duration) {
minDuration := leaseDuration
if minDuration > increment && increment > 0 {
minDuration = increment
}
if minDuration <= 0 {
r.grace = 0
return
}
leaseNanos := float64(minDuration.Nanoseconds())
jitterMax := 0.1 * leaseNanos
// For a given lease duration, we want to allow 80-90% of that to elapse,
// so the remaining amount is the grace period
r.grace = time.Duration(jitterMax) + time.Duration(uint64(r.random.Int63())%uint64(jitterMax))
}
type (
Renewer = LifetimeWatcher
RenewerInput = LifetimeWatcherInput
)