vendor: update all dependencies
This commit is contained in:
Nick Craig-Wood
+234
-115
@@ -17,32 +17,42 @@ package pubsub
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/iam"
|
||||
"golang.org/x/net/context"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
)
|
||||
|
||||
// The default period for which to automatically extend Message acknowledgement deadlines.
|
||||
const DefaultMaxExtension = 10 * time.Minute
|
||||
|
||||
// The default maximum number of messages that are prefetched from the server.
|
||||
const DefaultMaxPrefetch = 100
|
||||
|
||||
// Subscription is a reference to a PubSub subscription.
|
||||
type Subscription struct {
|
||||
s service
|
||||
|
||||
// The fully qualified identifier for the subscription, in the format "projects/<projid>/subscriptions/<name>"
|
||||
name string
|
||||
|
||||
// Settings for pulling messages. Configure these before calling Receive.
|
||||
ReceiveSettings ReceiveSettings
|
||||
|
||||
mu sync.Mutex
|
||||
receiveActive bool
|
||||
}
|
||||
|
||||
// Subscription creates a reference to a subscription.
|
||||
func (c *Client) Subscription(id string) *Subscription {
|
||||
return newSubscription(c.s, fmt.Sprintf("projects/%s/subscriptions/%s", c.projectID, id))
|
||||
}
|
||||
|
||||
func newSubscription(s service, name string) *Subscription {
|
||||
return &Subscription{
|
||||
s: c.s,
|
||||
name: fmt.Sprintf("projects/%s/subscriptions/%s", c.projectID, id),
|
||||
s: s,
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +91,7 @@ func (subs *SubscriptionIterator) Next() (*Subscription, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Subscription{s: subs.s, name: subName}, nil
|
||||
return newSubscription(subs.s, subName), nil
|
||||
}
|
||||
|
||||
// PushConfig contains configuration for subscriptions that operate in push mode.
|
||||
@@ -100,9 +110,58 @@ type SubscriptionConfig struct {
|
||||
|
||||
// The default maximum time after a subscriber receives a message before
|
||||
// the subscriber should acknowledge the message. Note: messages which are
|
||||
// obtained via a MessageIterator need not be acknowledged within this
|
||||
// obtained via Subscription.Receive need not be acknowledged within this
|
||||
// deadline, as the deadline will be automatically extended.
|
||||
AckDeadline time.Duration
|
||||
|
||||
// Whether to retain acknowledged messages. If true, acknowledged messages
|
||||
// will not be expunged until they fall out of the RetentionDuration window.
|
||||
retainAckedMessages bool
|
||||
|
||||
// How long to retain messages in backlog, from the time of publish. If RetainAckedMessages is true,
|
||||
// this duration affects the retention of acknowledged messages,
|
||||
// otherwise only unacknowledged messages are retained.
|
||||
// Defaults to 7 days. Cannot be longer than 7 days or shorter than 10 minutes.
|
||||
retentionDuration time.Duration
|
||||
}
|
||||
|
||||
// ReceiveSettings configure the Receive method.
|
||||
// A zero ReceiveSettings will result in values equivalent to DefaultReceiveSettings.
|
||||
type ReceiveSettings struct {
|
||||
// MaxExtension is the maximum period for which the Subscription should
|
||||
// automatically extend the ack deadline for each message.
|
||||
//
|
||||
// The Subscription will automatically extend the ack deadline of all
|
||||
// fetched Messages for the duration specified. Automatic deadline
|
||||
// extension may be disabled by specifying a duration less than 1.
|
||||
MaxExtension time.Duration
|
||||
|
||||
// MaxOutstandingMessages is the maximum number of unprocessed messages
|
||||
// (unacknowledged but not yet expired). If MaxOutstandingMessages is 0, it
|
||||
// will be treated as if it were DefaultReceiveSettings.MaxOutstandingMessages.
|
||||
// If the value is negative, then there will be no limit on the number of
|
||||
// unprocessed messages.
|
||||
MaxOutstandingMessages int
|
||||
|
||||
// MaxOutstandingBytes is the maximum size of unprocessed messages
|
||||
// (unacknowledged but not yet expired). If MaxOutstandingBytes is 0, it will
|
||||
// be treated as if it were DefaultReceiveSettings.MaxOutstandingBytes. If
|
||||
// the value is negative, then there will be no limit on the number of bytes
|
||||
// for unprocessed messages.
|
||||
MaxOutstandingBytes int
|
||||
|
||||
// NumGoroutines is the number of goroutines Receive will spawn to pull
|
||||
// messages concurrently. If NumGoroutines is less than 1, it will be treated
|
||||
// as if it were DefaultReceiveSettings.NumGoroutines.
|
||||
NumGoroutines int
|
||||
}
|
||||
|
||||
// DefaultReceiveSettings holds the default values for ReceiveSettings.
|
||||
var DefaultReceiveSettings = ReceiveSettings{
|
||||
MaxExtension: 10 * time.Minute,
|
||||
MaxOutstandingMessages: 1000,
|
||||
MaxOutstandingBytes: 1e9, // 1G
|
||||
NumGoroutines: 10 * runtime.GOMAXPROCS(0),
|
||||
}
|
||||
|
||||
// Delete deletes the subscription.
|
||||
@@ -116,10 +175,10 @@ func (s *Subscription) Exists(ctx context.Context) (bool, error) {
|
||||
}
|
||||
|
||||
// Config fetches the current configuration for the subscription.
|
||||
func (s *Subscription) Config(ctx context.Context) (*SubscriptionConfig, error) {
|
||||
func (s *Subscription) Config(ctx context.Context) (SubscriptionConfig, error) {
|
||||
conf, topicName, err := s.s.getSubscriptionConfig(ctx, s.name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return SubscriptionConfig{}, err
|
||||
}
|
||||
conf.Topic = &Topic{
|
||||
s: s.s,
|
||||
@@ -128,138 +187,198 @@ func (s *Subscription) Config(ctx context.Context) (*SubscriptionConfig, error)
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
// Pull returns a MessageIterator that can be used to fetch Messages. The MessageIterator
|
||||
// will automatically extend the ack deadline of all fetched Messages, for the
|
||||
// period specified by DefaultMaxExtension. This may be overridden by supplying
|
||||
// a MaxExtension pull option.
|
||||
//
|
||||
// If ctx is cancelled or exceeds its deadline, outstanding acks or deadline
|
||||
// extensions will fail.
|
||||
//
|
||||
// The caller must call Stop on the MessageIterator once finished with it.
|
||||
func (s *Subscription) Pull(ctx context.Context, opts ...PullOption) (*MessageIterator, error) {
|
||||
config, err := s.Config(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
po := processPullOptions(opts)
|
||||
po.ackDeadline = config.AckDeadline
|
||||
return newMessageIterator(ctx, s.s, s.name, po), nil
|
||||
// SubscriptionConfigToUpdate describes how to update a subscription.
|
||||
type SubscriptionConfigToUpdate struct {
|
||||
// If non-nil, the push config is changed.
|
||||
PushConfig *PushConfig
|
||||
}
|
||||
|
||||
// ModifyPushConfig updates the endpoint URL and other attributes of a push subscription.
|
||||
func (s *Subscription) ModifyPushConfig(ctx context.Context, conf *PushConfig) error {
|
||||
if conf == nil {
|
||||
return errors.New("must supply non-nil PushConfig")
|
||||
// Update changes an existing subscription according to the fields set in cfg.
|
||||
// It returns the new SubscriptionConfig.
|
||||
//
|
||||
// Update returns an error if no fields were modified.
|
||||
func (s *Subscription) Update(ctx context.Context, cfg SubscriptionConfigToUpdate) (SubscriptionConfig, error) {
|
||||
if cfg.PushConfig == nil {
|
||||
return SubscriptionConfig{}, errors.New("pubsub: UpdateSubscription call with nothing to update")
|
||||
}
|
||||
|
||||
return s.s.modifyPushConfig(ctx, s.name, conf)
|
||||
if err := s.s.modifyPushConfig(ctx, s.name, *cfg.PushConfig); err != nil {
|
||||
return SubscriptionConfig{}, err
|
||||
}
|
||||
return s.Config(ctx)
|
||||
}
|
||||
|
||||
func (s *Subscription) IAM() *iam.Handle {
|
||||
return s.s.iamHandle(s.name)
|
||||
}
|
||||
|
||||
// A PullOption is an optional argument to Subscription.Pull.
|
||||
type PullOption interface {
|
||||
setOptions(o *pullOptions)
|
||||
}
|
||||
|
||||
type pullOptions struct {
|
||||
// maxExtension is the maximum period for which the iterator should
|
||||
// automatically extend the ack deadline for each message.
|
||||
maxExtension time.Duration
|
||||
|
||||
// maxPrefetch is the maximum number of Messages to have in flight, to
|
||||
// be returned by MessageIterator.Next.
|
||||
maxPrefetch int32
|
||||
|
||||
// ackDeadline is the default ack deadline for the subscription. Not
|
||||
// configurable via a PullOption.
|
||||
ackDeadline time.Duration
|
||||
}
|
||||
|
||||
func processPullOptions(opts []PullOption) *pullOptions {
|
||||
po := &pullOptions{
|
||||
maxExtension: DefaultMaxExtension,
|
||||
maxPrefetch: DefaultMaxPrefetch,
|
||||
}
|
||||
|
||||
for _, o := range opts {
|
||||
o.setOptions(po)
|
||||
}
|
||||
|
||||
return po
|
||||
}
|
||||
|
||||
type maxPrefetch int32
|
||||
|
||||
func (max maxPrefetch) setOptions(o *pullOptions) {
|
||||
if o.maxPrefetch = int32(max); o.maxPrefetch < 1 {
|
||||
o.maxPrefetch = 1
|
||||
}
|
||||
}
|
||||
|
||||
// MaxPrefetch returns a PullOption that limits Message prefetching.
|
||||
//
|
||||
// For performance reasons, the pubsub library may prefetch a pool of Messages
|
||||
// to be returned serially from MessageIterator.Next. MaxPrefetch is used to limit the
|
||||
// the size of this pool.
|
||||
//
|
||||
// If num is less than 1, it will be treated as if it were 1.
|
||||
func MaxPrefetch(num int) PullOption {
|
||||
return maxPrefetch(trunc32(int64(num)))
|
||||
}
|
||||
|
||||
type maxExtension time.Duration
|
||||
|
||||
func (max maxExtension) setOptions(o *pullOptions) {
|
||||
if o.maxExtension = time.Duration(max); o.maxExtension < 0 {
|
||||
o.maxExtension = 0
|
||||
}
|
||||
}
|
||||
|
||||
// MaxExtension returns a PullOption that limits how long acks deadlines are
|
||||
// extended for.
|
||||
//
|
||||
// A MessageIterator will automatically extend the ack deadline of all fetched
|
||||
// Messages for the duration specified. Automatic deadline extension may be
|
||||
// disabled by specifying a duration of 0.
|
||||
func MaxExtension(duration time.Duration) PullOption {
|
||||
return maxExtension(duration)
|
||||
}
|
||||
|
||||
// CreateSubscription creates a new subscription on a topic.
|
||||
//
|
||||
// name is the name of the subscription to create. It must start with a letter,
|
||||
// id is the name of the subscription to create. It must start with a letter,
|
||||
// and contain only letters ([A-Za-z]), numbers ([0-9]), dashes (-),
|
||||
// underscores (_), periods (.), tildes (~), plus (+) or percent signs (%). It
|
||||
// must be between 3 and 255 characters in length, and must not start with
|
||||
// "goog".
|
||||
//
|
||||
// topic is the topic from which the subscription should receive messages. It
|
||||
// need not belong to the same project as the subscription.
|
||||
// cfg.Topic is the topic from which the subscription should receive messages. It
|
||||
// need not belong to the same project as the subscription. This field is required.
|
||||
//
|
||||
// ackDeadline is the maximum time after a subscriber receives a message before
|
||||
// cfg.AckDeadline is the maximum time after a subscriber receives a message before
|
||||
// the subscriber should acknowledge the message. It must be between 10 and 600
|
||||
// seconds (inclusive), and is rounded down to the nearest second. If the
|
||||
// provided ackDeadline is 0, then the default value of 10 seconds is used.
|
||||
// Note: messages which are obtained via a MessageIterator need not be
|
||||
// Note: messages which are obtained via Subscription.Receive need not be
|
||||
// acknowledged within this deadline, as the deadline will be automatically
|
||||
// extended.
|
||||
//
|
||||
// pushConfig may be set to configure this subscription for push delivery.
|
||||
// cfg.PushConfig may be set to configure this subscription for push delivery.
|
||||
//
|
||||
// If the subscription already exists an error will be returned.
|
||||
func (c *Client) CreateSubscription(ctx context.Context, id string, topic *Topic, ackDeadline time.Duration, pushConfig *PushConfig) (*Subscription, error) {
|
||||
if ackDeadline == 0 {
|
||||
ackDeadline = 10 * time.Second
|
||||
func (c *Client) CreateSubscription(ctx context.Context, id string, cfg SubscriptionConfig) (*Subscription, error) {
|
||||
if cfg.Topic == nil {
|
||||
return nil, errors.New("pubsub: require non-nil Topic")
|
||||
}
|
||||
if d := ackDeadline.Seconds(); d < 10 || d > 600 {
|
||||
if cfg.AckDeadline == 0 {
|
||||
cfg.AckDeadline = 10 * time.Second
|
||||
}
|
||||
if d := cfg.AckDeadline; d < 10*time.Second || d > 600*time.Second {
|
||||
return nil, fmt.Errorf("ack deadline must be between 10 and 600 seconds; got: %v", d)
|
||||
}
|
||||
|
||||
sub := c.Subscription(id)
|
||||
err := c.s.createSubscription(ctx, topic.name, sub.name, ackDeadline, pushConfig)
|
||||
err := c.s.createSubscription(ctx, sub.name, cfg)
|
||||
return sub, err
|
||||
}
|
||||
|
||||
var errReceiveInProgress = errors.New("pubsub: Receive already in progress for this subscription")
|
||||
|
||||
// Receive calls f with the outstanding messages from the subscription.
|
||||
// It blocks until ctx is done, or the service returns a non-retryable error.
|
||||
//
|
||||
// The standard way to terminate a Receive is to cancel its context:
|
||||
//
|
||||
// cctx, cancel := context.WithCancel(ctx)
|
||||
// err := sub.Receive(cctx, callback)
|
||||
// // Call cancel from callback, or another goroutine.
|
||||
//
|
||||
// If the service returns a non-retryable error, Receive returns that error after
|
||||
// all of the outstanding calls to f have returned. If ctx is done, Receive
|
||||
// returns either nil after all of the outstanding calls to f have returned and
|
||||
// all messages have been acknowledged or have expired.
|
||||
//
|
||||
// Receive calls f concurrently from multiple goroutines. It is encouraged to
|
||||
// process messages synchronously in f, even if that processing is relatively
|
||||
// time-consuming; Receive will spawn new goroutines for incoming messages,
|
||||
// limited by MaxOutstandingMessages and MaxOutstandingBytes in ReceiveSettings.
|
||||
//
|
||||
// The context passed to f will be canceled when ctx is Done or there is a
|
||||
// fatal service error.
|
||||
//
|
||||
// Receive will automatically extend the ack deadline of all fetched Messages for the
|
||||
// period specified by s.ReceiveSettings.MaxExtension.
|
||||
//
|
||||
// Each Subscription may have only one invocation of Receive active at a time.
|
||||
func (s *Subscription) Receive(ctx context.Context, f func(context.Context, *Message)) error {
|
||||
s.mu.Lock()
|
||||
if s.receiveActive {
|
||||
s.mu.Unlock()
|
||||
return errReceiveInProgress
|
||||
}
|
||||
s.receiveActive = true
|
||||
s.mu.Unlock()
|
||||
defer func() { s.mu.Lock(); s.receiveActive = false; s.mu.Unlock() }()
|
||||
|
||||
config, err := s.Config(ctx)
|
||||
if err != nil {
|
||||
if grpc.Code(err) == codes.Canceled {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
maxCount := s.ReceiveSettings.MaxOutstandingMessages
|
||||
if maxCount == 0 {
|
||||
maxCount = DefaultReceiveSettings.MaxOutstandingMessages
|
||||
}
|
||||
maxBytes := s.ReceiveSettings.MaxOutstandingBytes
|
||||
if maxBytes == 0 {
|
||||
maxBytes = DefaultReceiveSettings.MaxOutstandingBytes
|
||||
}
|
||||
maxExt := s.ReceiveSettings.MaxExtension
|
||||
if maxExt == 0 {
|
||||
maxExt = DefaultReceiveSettings.MaxExtension
|
||||
} else if maxExt < 0 {
|
||||
// If MaxExtension is negative, disable automatic extension.
|
||||
maxExt = 0
|
||||
}
|
||||
numGoroutines := s.ReceiveSettings.NumGoroutines
|
||||
if numGoroutines < 1 {
|
||||
numGoroutines = DefaultReceiveSettings.NumGoroutines
|
||||
}
|
||||
// TODO(jba): add tests that verify that ReceiveSettings are correctly processed.
|
||||
po := &pullOptions{
|
||||
maxExtension: maxExt,
|
||||
maxPrefetch: trunc32(int64(maxCount)),
|
||||
ackDeadline: config.AckDeadline,
|
||||
}
|
||||
fc := newFlowController(maxCount, maxBytes)
|
||||
|
||||
// Wait for all goroutines started by Receive to return, so instead of an
|
||||
// obscure goroutine leak we have an obvious blocked call to Receive.
|
||||
group, gctx := errgroup.WithContext(ctx)
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
group.Go(func() error {
|
||||
return s.receive(gctx, group, po, fc, f)
|
||||
})
|
||||
}
|
||||
return group.Wait()
|
||||
}
|
||||
|
||||
func (s *Subscription) receive(ctx context.Context, group *errgroup.Group, po *pullOptions, fc *flowController, f func(context.Context, *Message)) error {
|
||||
// Cancel a sub-context when we return, to kick the context-aware callbacks
|
||||
// and the goroutine below.
|
||||
ctx2, cancel := context.WithCancel(ctx)
|
||||
// Call stop when Receive's context is done.
|
||||
// Stop will block until all outstanding messages have been acknowledged
|
||||
// or there was a fatal service error.
|
||||
// The iterator does not use the context passed to Receive. If it did, canceling
|
||||
// that context would immediately stop the iterator without waiting for unacked
|
||||
// messages.
|
||||
iter := newMessageIterator(context.Background(), s.s, s.name, po)
|
||||
group.Go(func() error {
|
||||
<-ctx2.Done()
|
||||
iter.Stop()
|
||||
return nil
|
||||
})
|
||||
defer cancel()
|
||||
for {
|
||||
msg, err := iter.Next()
|
||||
if err == iterator.Done {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO(jba): call acquire closer to when the message is allocated.
|
||||
if err := fc.acquire(ctx, len(msg.Data)); err != nil {
|
||||
// TODO(jba): test that this "orphaned" message is nacked immediately when ctx is done.
|
||||
msg.Nack()
|
||||
return nil
|
||||
}
|
||||
group.Go(func() error {
|
||||
// TODO(jba): call release when the message is available for GC.
|
||||
// This considers the message to be released when
|
||||
// f is finished, but f may ack early or not at all.
|
||||
defer fc.release(len(msg.Data))
|
||||
f(ctx2, msg)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(jba): remove when we delete messageIterator.
|
||||
type pullOptions struct {
|
||||
maxExtension time.Duration
|
||||
maxPrefetch int32
|
||||
// ackDeadline is the default ack deadline for the subscription. Not
|
||||
// configurable.
|
||||
ackDeadline time.Duration
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user