1
//! Backend logic to upload a document to multiple targets
2
use std::{
3
    collections::HashMap,
4
    fmt::Debug,
5
    hash::Hash,
6
    num::NonZeroUsize,
7
    sync::{
8
        Arc,
9
        atomic::{self, AtomicUsize},
10
    },
11
    time::Duration,
12
};
13

            
14
use futures::{
15
    FutureExt as _, StreamExt as _, future::BoxFuture, select_biased, stream::Fuse,
16
    stream::FuturesUnordered,
17
};
18
use postage::watch;
19
use tor_basic_utils::retry::RetryDelay;
20
use tor_error::warn_report;
21
use tor_rtcompat::SleepProvider;
22
use tracing::{Level, debug, span, trace, warn};
23
use web_time_compat::Instant;
24

            
25
use crate::{
26
    DocVersion, Document, PublishDirective, PublishStatus, Rejection, UploadError, Uploader,
27
};
28

            
29
/// Identifier for a single action that we have queued for a target.
30
///
31
/// (Actions can currently be "try to upload" or "wait till later.")
32
#[derive(Eq, PartialEq, Clone, Copy, Debug)]
33
struct ActionNum(NonZeroUsize);
34

            
35
impl ActionNum {
36
    /// Return a new identifier.
37
    ///
38
    /// We do not guarantee that these are permanently unique:
39
    /// only that there are very unlikely to be two actions with the same ActionNum
40
    /// active at once.
41
88
    fn next() -> Self {
42
        static NEXT: AtomicUsize = AtomicUsize::new(0);
43

            
44
        loop {
45
90
            let val = NEXT.fetch_add(1, atomic::Ordering::Relaxed);
46
90
            if let Some(nz) = NonZeroUsize::new(val) {
47
88
                return ActionNum(nz);
48
2
            }
49
        }
50
88
    }
51
}
52

            
53
/// Mutable status for a single target.
54
#[derive(Debug)]
55
struct TargetStatus {
56
    /// The current state of this target.
57
    state: TargetState,
58

            
59
    /// How many times has this target failed since we last received an answer from it?
60
    /// ("Published" and "Rejected" both count as answers; "Try again later" does not.)
61
    n_failures: usize,
62

            
63
    /// State of our retry-timing algorithm.
64
    retry: RetryDelay,
65

            
66
    /// Identifier for the most recent action that we launched for this target.
67
    ///
68
    /// (Multiple actions can be pending at once.
69
    /// We call an action "the latest action for a target"
70
    /// if its ActionNum matches this value.)
71
    latest_action: Option<ActionNum>,
72
}
73

            
74
/// The current state for a single target.
75
#[derive(Debug)]
76
#[expect(unused)] // TODO: we don't use all these fields yet; we should remove or expose them.
77
enum TargetState {
78
    /// There is no document to upload, so we don't have anything to do.
79
    NoDocument,
80

            
81
    /// We are ready to try uploading the most recent document to this target.
82
    Ready,
83

            
84
    /// We are trying to upload the most recent document to this target.
85
    ///
86
    /// (Invariant: If a target is in this state, there is an Upload future
87
    /// among our pending actions for that target.)
88
    Inflight {
89
        /// The time when we began trying to upload.
90
        since: Instant,
91
    },
92

            
93
    /// An upload attempt has failed; we are waiting for a while before we try again.
94
    ///
95
    /// (Invariant: If a target is in this state, there is a Sleep future
96
    /// among our pending actions for that target.)
97
    Waiting {
98
        /// The time until which we are waiting.
99
        until: Instant,
100
    },
101

            
102
    /// We have published the most recent document to this target.
103
    Published,
104

            
105
    /// We have failed permanently for some reason.
106
    /// We won't retry until the document changes.
107
    PermanentlyFailed(UploadError),
108

            
109
    /// The target told us that it will not accept the most recent document,
110
    /// and so we should not try that document again.
111
    Rejected(Rejection),
112
}
113

            
114
/// An enum to declare whether a document is present.
115
//
116
// (We don't use bool here because bools are error-prone.)
117
#[derive(Clone, Copy, Debug)]
118
enum DocumentPresent {
119
    /// We have a document.
120
    Present,
121
    /// We do not currently have a document to upload
122
    Absent,
123
}
124

            
125
impl DocumentPresent {
126
    /// Return the initial state that new targets should enter.
127
28
    fn initial_target_state(self) -> TargetState {
128
28
        match self {
129
16
            DocumentPresent::Present => TargetState::Ready,
130
12
            DocumentPresent::Absent => TargetState::NoDocument,
131
        }
132
28
    }
133
}
134

            
135
impl<D: ?Sized> super::Document<D> {
136
    /// Return a DocumentPresent for this document.
137
24
    fn present(&self) -> DocumentPresent {
138
24
        if self.contents.is_some() {
139
20
            DocumentPresent::Present
140
        } else {
141
4
            DocumentPresent::Absent
142
        }
143
24
    }
144
}
145

            
146
impl TargetStatus {
147
    /// Construct a new TargetStatus in the Ready state.
148
28
    fn new(initial_delay: Duration, document_present: DocumentPresent) -> Self {
149
28
        Self {
150
28
            n_failures: 0,
151
28
            retry: RetryDelay::from_duration(initial_delay),
152
28
            latest_action: None,
153
28
            state: document_present.initial_target_state(),
154
28
        }
155
28
    }
156

            
157
    /// Called after the latest action for a target has failed:
158
    /// increments the failure count, and returns the interval for which we should wait.
159
    ///
160
    /// If `suggested_delay` is provided, it is an amount that the target
161
    /// told us to wait before retrying.   We will treat this amount
162
    /// as a _minimum_.  A `suggested_delay` will not prevent us
163
    /// from retrying immediately if our failure status is reset.
164
24
    fn set_waiting(
165
24
        &mut self,
166
24
        now: Instant,
167
24
        suggested_delay: Option<Duration>,
168
24
        action: ActionNum,
169
24
    ) -> Duration {
170
24
        self.n_failures += 1;
171
24
        self.latest_action = Some(action);
172

            
173
24
        let d = self.retry.next_delay(&mut rand::rng());
174
24
        let suggested = suggested_delay.unwrap_or_default();
175
24
        let d = std::cmp::max(d, suggested);
176

            
177
24
        self.state = TargetState::Waiting { until: now + d };
178
24
        d
179
24
    }
180

            
181
    /// Called after we have decided to launch an upload for a target.
182
64
    fn set_inflight(&mut self, now: Instant, action: ActionNum) {
183
64
        self.state = TargetState::Inflight { since: now };
184
64
        self.latest_action = Some(action);
185
64
    }
186

            
187
    /// Change this action's state to Ready.
188
24
    fn set_ready(&mut self) {
189
24
        self.state = TargetState::Ready;
190
24
    }
191

            
192
    /// Reset the failure count and timeout for this target.
193
40
    fn reset_failures(&mut self) {
194
40
        self.n_failures = 0;
195
40
        self.retry.reset();
196
40
    }
197

            
198
    /// Called after the latest action for a target has succeeded
199
    /// in uploading the most recent document.
200
40
    fn set_published(&mut self) {
201
40
        self.reset_failures();
202
40
        self.state = TargetState::Published;
203
40
    }
204

            
205
    /// Called after the latest action for a target has been rejected
206
    /// in uploading the most recent document.
207
    fn set_rejected(&mut self, rejection: Rejection) {
208
        self.reset_failures();
209
        self.state = TargetState::Rejected(rejection);
210
    }
211

            
212
    /// Mark this target as permanently unable to receive the current
213
    /// document because of some error `e`.
214
    fn set_permanently_failed(&mut self, e: UploadError) {
215
        self.state = TargetState::PermanentlyFailed(e);
216
    }
217
}
218

            
219
/// The result of a single action.
220
#[derive(Debug)]
221
enum ActionOutcome {
222
    /// A sleep action has expired.
223
    DoneSleeping,
224

            
225
    /// An upload action has succeeded.
226
    Published,
227

            
228
    /// An upload action has been rejected.
229
    Rejected(Rejection),
230

            
231
    /// An upload action has failed with some error.
232
    Err(UploadError),
233
}
234

            
235
impl ActionOutcome {
236
    /// Construct an [`ActionOutcome`] from the result of [`Uploader::upload()`].
237
64
    fn from_upload_result(r: Result<(), UploadError>) -> Self {
238
24
        match r {
239
40
            Ok(()) => Self::Published,
240
            Err(UploadError::Rejected(rejection)) => Self::Rejected(rejection),
241
24
            Err(e) => Self::Err(e),
242
        }
243
64
    }
244
}
245

            
246
/// The type returned by one of the action futures in `PublishReactor.inflight`.
247
struct TaskResult<T: ?Sized> {
248
    /// Which target were we taking this action for?
249
    target: Arc<T>,
250
    /// An `ActionNum` to identify whether the action was the latest one for the target.
251
    action: ActionNum,
252
    /// Which document was the most recent when the action was launched?
253
    doc_version: DocVersion,
254
    /// The result of the action.
255
    outcome: ActionOutcome,
256
}
257

            
258
/// Backend data that we use to publish a document (or series of documents).
259
pub(crate) struct PublishReactor<R: SleepProvider, D: ?Sized, T, UP: ?Sized>
260
where
261
    T: Hash + Eq + ?Sized,
262
{
263
    /// A sleep provider used to launch wait actions.
264
    runtime: R,
265

            
266
    /// A description of what we're uploading, for log messages.
267
    description: String,
268

            
269
    /// The current actions that the [`Publisher`](crate::Publisher) has told us to take.
270
    ///
271
    /// We watch for changes in this directive and adjust our behavior accordingly.
272
    directive: Fuse<watch::Receiver<PublishDirective<D, T>>>,
273

            
274
    /// A channel we use to report our current status to the [`Publisher`](crate::Publisher).
275
    status: watch::Sender<PublishStatus>,
276

            
277
    /// The document which we are currently trying to upload.
278
    latest_document: Document<D>,
279

            
280
    /// The initial retry delay for a failed target.
281
    ///
282
    /// (Used to seed our retry delay algorithm.)
283
    initial_retry_delay: Duration,
284

            
285
    /// The most recent value we've seen for the `reset_count` field of our `PublishDirective`.
286
    latest_reset_count: usize,
287

            
288
    /// The [`Uploader`] object we use to upload documents.
289
    uploader: Arc<UP>,
290

            
291
    /// A set of all our pending actions.
292
    ///
293
    /// Actions are either upload attempts, or sleep actions.
294
    ///
295
    /// Additionally, this `FuturesUnordered` contains a single future that is always
296
    /// pending, to guarantee that `inflight.next()` never returns None.
297
    ///
298
    /// Multiple actions may be inflight for a given target at a time.
299
    /// This is deliberate:
300
    /// If we change our document while the upload of an older document is inflight,
301
    /// we do not want to stop the inflight upload in the middle.
302
    ///
303
    /// TODO: We _could_ cancel any Sleep action that is superseded.
304
    /// That's a fair amount of effort, though, since FuturesUnordered doesn't have
305
    /// very nice accessors nor does Sleep have a good way to make it cancellable.
306
    inflight: FuturesUnordered<BoxFuture<'static, TaskResult<T>>>,
307

            
308
    /// The current status for each of our live targets.
309
    target_status: HashMap<Arc<T>, TargetStatus>,
310
}
311

            
312
/// Return type used to tell the reactor loop to exit.
313
#[derive(Debug)]
314
struct ExitLoop;
315

            
316
impl<R: SleepProvider, D, T, UP> PublishReactor<R, D, T, UP>
317
where
318
    D: ?Sized + Send + Sync + 'static,
319
    T: Hash + Eq + Send + Sync + Debug + ?Sized + 'static,
320
    UP: Uploader<Doc = D, Target = T> + ?Sized,
321
{
322
    /// Construct a new reactor.
323
    ///
324
    /// (Does not launch any background task or do any work).
325
8
    pub(crate) fn new(
326
8
        runtime: R,
327
8
        description: String,
328
8
        action: watch::Receiver<PublishDirective<D, T>>,
329
8
        status: watch::Sender<PublishStatus>,
330
8
        initial_retry_delay: Duration,
331
8
        publisher: Arc<UP>,
332
8
    ) -> Self {
333
8
        let (latest_document, latest_reset_count, targets) = {
334
8
            let cur_action = action.borrow();
335
8
            (
336
8
                cur_action.document.clone(),
337
8
                cur_action.reset_failures_count,
338
8
                cur_action.targets.clone(),
339
8
            )
340
8
        };
341

            
342
8
        let inflight = FuturesUnordered::new();
343
        // Add a never-finished future to keep FuturesUnordered from saying it's done.
344
8
        inflight.push(Box::pin(std::future::pending()) as _);
345
8
        let document_present = latest_document.present();
346

            
347
8
        let target_status = targets
348
8
            .into_iter()
349
24
            .map(|t| (t, TargetStatus::new(initial_retry_delay, document_present)))
350
8
            .collect();
351

            
352
8
        Self {
353
8
            runtime,
354
8
            description,
355
8
            directive: action.fuse(),
356
8
            status,
357
8
            latest_document,
358
8
            initial_retry_delay,
359
8
            latest_reset_count,
360
8
            uploader: publisher,
361
8
            inflight,
362
8
            target_status,
363
8
        }
364
8
    }
365

            
366
    /// Run forever, handling changes in the [`PublishDirective`], uploading documents, and reporting status.
367
8
    pub(crate) async fn run(mut self) {
368
8
        let _span = span!(Level::TRACE, "Publishing {}", self.description);
369

            
370
        // The first time we start, we begin uploading.
371
8
        self.launch_ready_requests(self.runtime.now());
372
8
        self.recalculate_status();
373

            
374
        'mainloop: loop {
375
112
            select_biased! {
376
                // We've been told to do something new, _or_ the last handle to the Publisher has
377
                // been dropped.
378
112
                directive_changed = self.directive.next() => {
379
24
                    let Some(directive) = directive_changed else {
380
                        // The watch::Receiver stream returned None,
381
                        // so we know that the last handle has been dropped.
382
8
                        trace!("directive stream dropped; exiting");
383
8
                        break 'mainloop;
384
                    };
385

            
386
                    // Process any change in the action.
387
16
                    if let Err(ExitLoop) = self.directive_changed(&directive) {
388
                        trace!("directive is shutdown: exiting");
389
                        break 'mainloop;
390
16
                    }
391

            
392
                    // Update our `PublishStatus`.
393
16
                    self.recalculate_status();
394
                }
395

            
396
                // Some action has finished; update accordingly.
397
112
                publication_result = self.inflight.next() => {
398
88
                    let task_result = publication_result.expect("Stream ended unexpectedly.");
399
88
                    self.handle_task_result(task_result);
400
88
                }
401
            }
402
        }
403

            
404
8
        self.status.borrow_mut().shutdown = true;
405
8
    }
406

            
407
    /// Called when a task in `self.inflight` produces a result.
408
    ///
409
    /// Update our status and launch new tasks as appropriate.
410
88
    fn handle_task_result(&mut self, task_result: TaskResult<T>) {
411
        let TaskResult {
412
88
            target,
413
88
            action,
414
88
            doc_version,
415
88
            outcome,
416
88
        } = task_result;
417

            
418
88
        let Some(status) = self.target_status.get_mut(&target) else {
419
            // The target isn't here, so we don't care about what happened with it.
420
            trace!(?target, ?outcome, "Ignoring result for removed target.");
421
            return;
422
        };
423
88
        if Some(action) != status.latest_action {
424
            // There is a more recent inflight action for this target;
425
            // ignore the results of this one.
426
            //
427
            // (See note on `Publish.inflight` about why we can have multiple inflight
428
            // actions.)
429
            //
430
            // We use a != comparison here rather than < since we allow the action
431
            // identifier space to wrap around.
432
            trace!(?target, ?outcome, "Ignoring result for superseded action.");
433
            return;
434
88
        }
435

            
436
24
        match outcome {
437
            ActionOutcome::Published => {
438
40
                if doc_version != self.latest_document.version {
439
                    // We aren't tracking this particular document any more;
440
                    // this was a stale upload.
441
                    return;
442
40
                }
443

            
444
40
                trace!(?target, "Document published");
445
40
                status.set_published();
446
            }
447
            ActionOutcome::Rejected(rejection) => {
448
                if doc_version != self.latest_document.version {
449
                    // We aren't tracking this particular document any more;
450
                    // this was a stale upload.
451
                    return;
452
                }
453

            
454
                warn!(
455
                    "{} upload rejected. The target ({:?}) said {}",
456
                    &self.description, &target, &rejection
457
                );
458

            
459
                status.set_rejected(rejection);
460
            }
461
24
            ActionOutcome::DoneSleeping => {
462
24
                // It's time to try a new upload to this target.
463
24
                self.launch_one(&target, self.runtime.now());
464
24
            }
465
24
            ActionOutcome::Err(e) if !e.is_retriable() => {
466
                warn_report!(
467
                    &e,
468
                    "Attempt to publish {} to {:?} failed. Not retriable.",
469
                    &self.description,
470
                    &target
471
                );
472
                status.set_permanently_failed(e);
473
            }
474
24
            ActionOutcome::Err(e) => {
475
                // We failed to upload: we log the error and wait until it's time to
476
                // retry.
477

            
478
                // TODO: This might need to be downgraded, but for now we'll leave it as-is.
479
24
                warn_report!(
480
                    &e,
481
                    "Attempt to publish {} to {:?} failed. We'll retry later.",
482
                    &self.description,
483
                    &target
484
                );
485
24
                self.begin_sleeping(target, e.suggested_delay(), self.runtime.now());
486
            }
487
        }
488

            
489
88
        self.recalculate_status();
490
88
    }
491

            
492
    /// Called when we have received a new [`PublishDirective`] from the publisher.
493
    ///
494
    /// Update the status of all of our targets, and launch new uploads as appropriate.
495
16
    fn directive_changed(&mut self, directive: &PublishDirective<D, T>) -> Result<(), ExitLoop> {
496
        use TargetState::*;
497

            
498
16
        if directive.shutdown {
499
            // We're supposed to shut down.  Just go ahead and do that.
500
            return Err(ExitLoop);
501
16
        }
502

            
503
        // Check to see if any targets have been added or removed;
504
        // update target_status accordingly.
505
16
        let document_present = directive.document.present();
506
52
        for new_target in directive.targets.iter() {
507
52
            self.target_status
508
52
                .entry(new_target.clone())
509
52
                .or_insert_with(|| TargetStatus::new(self.initial_retry_delay, document_present));
510
        }
511
16
        self.target_status
512
56
            .retain(|t, _| directive.targets.contains(t));
513

            
514
        // Have gotten a new document?  Have we been told to reset failing targets?
515
        //
516
        // (We use != rather than > here since we want to allow these counters to wrap around.)
517
16
        let document_changed = directive.document.version != self.latest_document.version;
518
16
        let reset_failing_targets = directive.reset_failures_count != self.latest_reset_count;
519
        // Update our own versions of the counters from the PublishDirective.
520
16
        if document_changed {
521
8
            self.latest_document = directive.document.clone();
522
8
        }
523
16
        self.latest_reset_count = directive.reset_failures_count;
524

            
525
16
        let no_document = directive.document.contents.is_none();
526
16
        if document_changed {
527
8
            let v = self.latest_document.version;
528
8
            if no_document {
529
                trace!("Publisher paused (version {v:?})");
530
            } else {
531
8
                trace!("New document (version {v:?})");
532
            }
533
8
        }
534

            
535
        // Reset failure timings if appropriate,
536
        // and mark targets ready if we want to launch a new upload to them.
537
52
        for status in self.target_status.values_mut() {
538
52
            if reset_failing_targets {
539
                status.reset_failures();
540
52
            }
541

            
542
52
            if no_document {
543
                status.state = NoDocument;
544
                continue;
545
52
            }
546

            
547
52
            let should_reset = match &status.state {
548
                // If this target is waiting, then we should let it continue
549
                // waiting unless we've been told to reset failing targets.
550
                Waiting { .. } => reset_failing_targets,
551

            
552
                // If the target is ready, there's no point in making it ready.
553
4
                Ready => false,
554

            
555
                // If we're currently uploading to a target,
556
                // we only want to launch a new upload if the document changed.
557
12
                Inflight { .. } => document_changed,
558

            
559
                // If we've published successfully,
560
                // or if we have been rejected,
561
                // or if we had nothing to do,
562
                // we only want to launch a new upload if the document changed.
563
24
                Published | Rejected(_) | PermanentlyFailed(_) => document_changed,
564

            
565
                // If we had no document, we want to launch now that we have one.
566
12
                NoDocument => true,
567
            };
568

            
569
52
            if should_reset {
570
24
                status.set_ready();
571
28
            }
572
        }
573

            
574
16
        self.launch_ready_requests(self.runtime.now());
575

            
576
16
        Ok(())
577
16
    }
578

            
579
    /// Launch a new upload for every Ready target,
580
    /// making its status Inflight.
581
24
    fn launch_ready_requests(&mut self, now: Instant) {
582
        // Build a list of the ready targets.
583
        //
584
        // This is a separate step to avoid a concurrent mutable/immutable borrow.
585
24
        let to_launch: Vec<Arc<T>> = self
586
24
            .target_status
587
24
            .iter()
588
76
            .filter(|(_target, status)| matches!(&status.state, TargetState::Ready))
589
40
            .map(|(target, _status)| Arc::clone(target))
590
24
            .collect();
591

            
592
        // Launch an upload for each of them.
593
40
        for target in to_launch {
594
40
            self.launch_one(&target, now);
595
40
        }
596
24
    }
597

            
598
    /// Compute a new [`PublishStatus`] reflecting our progress uploading the current document,
599
    /// and deliver it to the Publisher.
600
112
    fn recalculate_status(&mut self) {
601
        use TargetState::*;
602

            
603
112
        let n_targets = self.target_status.len();
604
112
        let mut n_inert = 0;
605
112
        let mut n_pending = 0;
606
112
        let mut n_failing = 0;
607
112
        let mut n_failed = 0;
608
112
        let mut n_published = 0;
609
112
        let mut n_rejected = 0;
610

            
611
344
        for status in self.target_status.values() {
612
344
            match &status.state {
613
12
                NoDocument => n_inert += 1,
614
130
                Published => n_published += 1,
615
                Rejected(_) => n_rejected += 1,
616
                Ready => {}
617
                PermanentlyFailed(_) => n_failed += 1,
618
                Inflight { .. } | Waiting { .. } => {
619
202
                    if status.n_failures > 0 {
620
114
                        n_failing += 1;
621
114
                    } else {
622
88
                        n_pending += 1;
623
88
                    }
624
                }
625
            }
626
        }
627

            
628
112
        let new_status = PublishStatus {
629
112
            document_version: self.latest_document.version,
630
112
            n_targets,
631
112
            n_inert,
632
112
            n_published,
633
112
            n_rejected,
634
112
            n_failed_permanently: n_failed,
635
112
            n_failing,
636
112
            n_pending,
637
112
            initialized: true,
638
112
            shutdown: false,
639
112
        };
640
112
        debug!("Publishing {}: {}", &self.description, &new_status);
641

            
642
112
        {
643
112
            *self.status.borrow_mut() = new_status;
644
112
        }
645
112
    }
646

            
647
    /// Launch an upload action for a given `target`, changing its status to Inflight.
648
64
    fn launch_one(&mut self, target: &Arc<T>, now: Instant) {
649
        // Launch the publish request, and add it to inflight.
650
64
        let Some(status) = self.target_status.get_mut(target) else {
651
            return;
652
        };
653
64
        let Some(document) = &self.latest_document.contents else {
654
            // There's no document, so we can't upload it.
655
            return;
656
        };
657

            
658
64
        trace!(?target, "Launching {} upload request", &self.description);
659

            
660
64
        let doc_version = self.latest_document.version;
661
64
        let action = ActionNum::next();
662

            
663
64
        let uploader = Arc::clone(&self.uploader);
664
64
        let target = Arc::clone(target);
665
64
        let document = Arc::clone(document);
666
64
        let future = async move {
667
64
            uploader
668
64
                .upload(Arc::clone(&target), document)
669
64
                .map(move |res| TaskResult {
670
64
                    target,
671
64
                    action,
672
64
                    doc_version,
673
64
                    outcome: ActionOutcome::from_upload_result(res),
674
64
                })
675
64
                .await
676
64
        };
677
64
        self.inflight.push(Box::pin(future));
678

            
679
64
        status.set_inflight(now, action);
680
64
    }
681

            
682
    /// Launch a sleep action for a given `target`, changing its status to `Waiting`.
683
24
    fn begin_sleeping(&mut self, target: Arc<T>, suggested_delay: Option<Duration>, now: Instant) {
684
        // Launch the publish request, and add it to inflight.
685
24
        let Some(status) = self.target_status.get_mut(&target) else {
686
            return;
687
        };
688

            
689
24
        let action = ActionNum::next();
690
24
        let delay = status.set_waiting(now, suggested_delay, action);
691

            
692
24
        trace!(
693
            ?target,
694
            ?delay,
695
            "Waiting for next {} upload attempt.",
696
            &self.description
697
        );
698

            
699
24
        let doc_version = self.latest_document.version;
700
24
        let future = self.runtime.sleep(delay).map(move |()| TaskResult {
701
24
            target,
702
24
            action,
703
24
            doc_version,
704
24
            outcome: ActionOutcome::DoneSleeping,
705
24
        });
706
24
        self.inflight.push(Box::pin(future));
707
24
    }
708
}