async.c++ 28.8 KB
Newer Older
Kenton Varda's avatar
Kenton Varda committed
1 2
// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
3
//
Kenton Varda's avatar
Kenton Varda committed
4 5 6 7 8 9
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
10
//
Kenton Varda's avatar
Kenton Varda committed
11 12
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
13
//
Kenton Varda's avatar
Kenton Varda committed
14 15 16 17 18 19 20
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
21 22 23

#include "async.h"
#include "debug.h"
24
#include "vector.h"
25
#include "threadlocal.h"
26

Kenton Varda's avatar
Kenton Varda committed
27
#if KJ_USE_FUTEX
28 29 30
#include <unistd.h>
#include <sys/syscall.h>
#include <linux/futex.h>
31
#endif
32

33 34
#if !KJ_NO_RTTI
#include <typeinfo>
35 36
#if __GNUC__
#include <cxxabi.h>
37 38
#include <stdlib.h>
#endif
39 40
#endif

41 42 43 44
namespace kj {

namespace {

45
KJ_THREADLOCAL_PTR(EventLoop) threadLocalEventLoop = nullptr;
46

47
#define _kJ_ALREADY_READY reinterpret_cast< ::kj::_::Event*>(1)
48

49 50 51 52 53 54
EventLoop& currentEventLoop() {
  EventLoop* loop = threadLocalEventLoop;
  KJ_REQUIRE(loop != nullptr, "No event loop is running on this thread.");
  return *loop;
}

55
class BoolEvent: public _::Event {
56 57 58
public:
  bool fired = false;

59
  Maybe<Own<_::Event>> fire() override {
60
    fired = true;
61
    return nullptr;
62 63 64
  }
};

65 66
class YieldPromiseNode final: public _::PromiseNode {
public:
67 68
  void onReady(_::Event* event) noexcept override {
    if (event) event->armBreadthFirst();
69 70
  }
  void get(_::ExceptionOrValue& output) noexcept override {
Kenton Varda's avatar
Kenton Varda committed
71
    output.as<_::Void>() = _::Void();
72 73 74
  }
};

75
class NeverDonePromiseNode final: public _::PromiseNode {
76
public:
77
  void onReady(_::Event* event) noexcept override {
Kenton Varda's avatar
Kenton Varda committed
78
    // ignore
79 80 81 82 83 84
  }
  void get(_::ExceptionOrValue& output) noexcept override {
    KJ_FAIL_REQUIRE("Not ready.");
  }
};

85 86
}  // namespace

87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
// =======================================================================================

Canceler::~Canceler() noexcept(false) {
  cancel("operation canceled");
}

void Canceler::cancel(StringPtr cancelReason) {
  if (isEmpty()) return;
  cancel(Exception(Exception::Type::FAILED, __FILE__, __LINE__, kj::str(cancelReason)));
}

void Canceler::cancel(const Exception& exception) {
  for (;;) {
    KJ_IF_MAYBE(a, list) {
      list = a->next;
      a->prev = nullptr;
      a->next = nullptr;
      a->cancel(kj::cp(exception));
    } else {
      break;
    }
  }
}

void Canceler::release() {
  for (;;) {
    KJ_IF_MAYBE(a, list) {
      list = a->next;
      a->prev = nullptr;
      a->next = nullptr;
    } else {
      break;
    }
  }
}

Canceler::AdapterBase::AdapterBase(Canceler& canceler)
    : prev(canceler.list),
      next(canceler.list) {
  canceler.list = *this;
  KJ_IF_MAYBE(n, next) {
    n->prev = next;
  }
}

Canceler::AdapterBase::~AdapterBase() noexcept(false) {
  KJ_IF_MAYBE(p, prev) {
    *p = next;
  }
  KJ_IF_MAYBE(n, next) {
    n->prev = prev;
  }
}

141 142 143 144 145 146 147 148 149 150 151 152 153 154
Canceler::AdapterImpl<void>::AdapterImpl(kj::PromiseFulfiller<void>& fulfiller,
            Canceler& canceler, kj::Promise<void> inner)
    : AdapterBase(canceler),
      fulfiller(fulfiller),
      inner(inner.then(
          [&fulfiller]() { fulfiller.fulfill(); },
          [&fulfiller](kj::Exception&& e) { fulfiller.reject(kj::mv(e)); })
          .eagerlyEvaluate(nullptr)) {}

void Canceler::AdapterImpl<void>::cancel(kj::Exception&& e) {
  fulfiller.reject(kj::mv(e));
  inner = nullptr;
}

155 156
// =======================================================================================

157 158 159 160
TaskSet::TaskSet(TaskSet::ErrorHandler& errorHandler)
  : errorHandler(errorHandler) {}

TaskSet::~TaskSet() noexcept(false) {}
Kenton Varda's avatar
Kenton Varda committed
161

162
class TaskSet::Task final: public _::Event {
Kenton Varda's avatar
Kenton Varda committed
163
public:
164 165 166 167
  Task(TaskSet& taskSet, Own<_::PromiseNode>&& nodeParam)
      : taskSet(taskSet), node(kj::mv(nodeParam)) {
    node->setSelfPointer(&node);
    node->onReady(this);
Kenton Varda's avatar
Kenton Varda committed
168 169
  }

170 171
  Maybe<Own<Task>> next;
  Maybe<Own<Task>>* prev = nullptr;
Kenton Varda's avatar
Kenton Varda committed
172

173 174 175 176 177
protected:
  Maybe<Own<Event>> fire() override {
    // Get the result.
    _::ExceptionOr<_::Void> result;
    node->get(result);
Kenton Varda's avatar
Kenton Varda committed
178

179 180 181 182 183 184
    // Delete the node, catching any exceptions.
    KJ_IF_MAYBE(exception, kj::runCatchingExceptions([this]() {
      node = nullptr;
    })) {
      result.addException(kj::mv(*exception));
    }
185

186 187 188
    // Call the error handler if there was an exception.
    KJ_IF_MAYBE(e, result.exception) {
      taskSet.errorHandler.taskFailed(kj::mv(*e));
189 190
    }

191 192 193
    // Remove from the task list.
    KJ_IF_MAYBE(n, next) {
      n->get()->prev = prev;
Kenton Varda's avatar
Kenton Varda committed
194
    }
195 196 197 198 199
    Own<Event> self = kj::mv(KJ_ASSERT_NONNULL(*prev));
    KJ_ASSERT(self.get() == this);
    *prev = kj::mv(next);
    next = nullptr;
    prev = nullptr;
200 201 202 203 204 205 206 207

    KJ_IF_MAYBE(f, taskSet.emptyFulfiller) {
      if (taskSet.tasks == nullptr) {
        f->get()->fulfill();
        taskSet.emptyFulfiller = nullptr;
      }
    }

208 209 210 211 212 213
    return mv(self);
  }

  _::PromiseNode* getInnerForTrace() override {
    return node;
  }
Kenton Varda's avatar
Kenton Varda committed
214

215 216 217 218
private:
  TaskSet& taskSet;
  Own<_::PromiseNode> node;
};
Kenton Varda's avatar
Kenton Varda committed
219

220 221 222 223 224
void TaskSet::add(Promise<void>&& promise) {
  auto task = heap<Task>(*this, kj::mv(promise.node));
  KJ_IF_MAYBE(head, tasks) {
    head->get()->prev = &task->next;
    task->next = kj::mv(tasks);
225
  }
226 227 228
  task->prev = &tasks;
  tasks = kj::mv(task);
}
229

230 231 232 233 234 235 236 237 238 239
kj::String TaskSet::trace() {
  kj::Vector<kj::String> traces;

  Maybe<Own<Task>>* ptr = &tasks;
  for (;;) {
    KJ_IF_MAYBE(task, *ptr) {
      traces.add(task->get()->trace());
      ptr = &task->get()->next;
    } else {
      break;
240
    }
Kenton Varda's avatar
Kenton Varda committed
241 242
  }

243 244
  return kj::strArray(traces, "\n============================================\n");
}
Kenton Varda's avatar
Kenton Varda committed
245

246 247 248 249 250 251 252 253 254 255 256 257
Promise<void> TaskSet::onEmpty() {
  KJ_REQUIRE(emptyFulfiller == nullptr, "onEmpty() can only be called once at a time");

  if (tasks == nullptr) {
    return READY_NOW;
  } else {
    auto paf = newPromiseAndFulfiller<void>();
    emptyFulfiller = kj::mv(paf.fulfiller);
    return kj::mv(paf.promise);
  }
}

258
namespace _ {  // private
Kenton Varda's avatar
Kenton Varda committed
259 260 261 262 263 264 265 266 267 268 269 270

class LoggingErrorHandler: public TaskSet::ErrorHandler {
public:
  static LoggingErrorHandler instance;

  void taskFailed(kj::Exception&& exception) override {
    KJ_LOG(ERROR, "Uncaught exception in daemonized task.", exception);
  }
};

LoggingErrorHandler LoggingErrorHandler::instance = LoggingErrorHandler();

271 272
class NullEventPort: public EventPort {
public:
273
  bool wait() override {
274 275 276
    KJ_FAIL_REQUIRE("Nothing to wait for; this thread would hang forever.");
  }

277 278 279
  bool poll() override { return false; }

  void wake() const override {
280
    // TODO(someday): Implement using condvar.
281 282 283
    kj::throwRecoverableException(KJ_EXCEPTION(UNIMPLEMENTED,
        "Cross-thread events are not yet implemented for EventLoops with no EventPort."));
  }
284 285 286 287 288 289

  static NullEventPort instance;
};

NullEventPort NullEventPort::instance = NullEventPort();

Kenton Varda's avatar
Kenton Varda committed
290 291 292 293
}  // namespace _ (private)

// =======================================================================================

294
void EventPort::setRunnable(bool runnable) {}
295

296 297 298 299 300
void EventPort::wake() const {
  kj::throwRecoverableException(KJ_EXCEPTION(UNIMPLEMENTED,
      "cross-thread wake() not implemented by this EventPort implementation"));
}

301 302
EventLoop::EventLoop()
    : port(_::NullEventPort::instance),
303
      daemons(kj::heap<TaskSet>(_::LoggingErrorHandler::instance)) {}
304

305 306
EventLoop::EventLoop(EventPort& port)
    : port(port),
307
      daemons(kj::heap<TaskSet>(_::LoggingErrorHandler::instance)) {}
Kenton Varda's avatar
Kenton Varda committed
308

309
EventLoop::~EventLoop() noexcept(false) {
310 311 312 313 314 315 316 317 318
  // Destroy all "daemon" tasks, noting that their destructors might try to access the EventLoop
  // some more.
  daemons = nullptr;

  // The application _should_ destroy everything using the EventLoop before destroying the
  // EventLoop itself, so if there are events on the loop, this indicates a memory leak.
  KJ_REQUIRE(head == nullptr, "EventLoop destroyed with events still in the queue.  Memory leak?",
             head->trace()) {
    // Unlink all the events and hope that no one ever fires them...
319
    _::Event* event = head;
320
    while (event != nullptr) {
321
      _::Event* next = event->next;
322 323 324 325 326 327
      event->next = nullptr;
      event->prev = nullptr;
      event = next;
    }
    break;
  }
328 329 330 331 332 333

  KJ_REQUIRE(threadLocalEventLoop != this,
             "EventLoop destroyed while still current for the thread.") {
    threadLocalEventLoop = nullptr;
    break;
  }
334
}
335

336
void EventLoop::run(uint maxTurnCount) {
337 338 339
  running = true;
  KJ_DEFER(running = false);

340 341 342 343 344
  for (uint i = 0; i < maxTurnCount; i++) {
    if (!turn()) {
      break;
    }
  }
345

346
  setRunnable(isRunnable());
347 348
}

349 350
bool EventLoop::turn() {
  _::Event* event = head;
351

352 353 354 355 356 357 358 359
  if (event == nullptr) {
    // No events in the queue.
    return false;
  } else {
    head = event->next;
    if (head != nullptr) {
      head->prev = &head;
    }
360

361 362 363 364
    depthFirstInsertPoint = &head;
    if (tail == &event->next) {
      tail = &head;
    }
365

366 367
    event->next = nullptr;
    event->prev = nullptr;
368

369 370 371 372 373 374
    Maybe<Own<_::Event>> eventToDestroy;
    {
      event->firing = true;
      KJ_DEFER(event->firing = false);
      eventToDestroy = event->fire();
    }
Kenton Varda's avatar
Kenton Varda committed
375

376 377 378 379
    depthFirstInsertPoint = &head;
    return true;
  }
}
380

381 382 383 384
bool EventLoop::isRunnable() {
  return head != nullptr;
}

385 386 387 388 389 390 391
void EventLoop::setRunnable(bool runnable) {
  if (runnable != lastRunnableState) {
    port.setRunnable(runnable);
    lastRunnableState = runnable;
  }
}

392 393 394 395 396 397 398 399 400 401 402 403 404
void EventLoop::enterScope() {
  KJ_REQUIRE(threadLocalEventLoop == nullptr, "This thread already has an EventLoop.");
  threadLocalEventLoop = this;
}

void EventLoop::leaveScope() {
  KJ_REQUIRE(threadLocalEventLoop == this,
             "WaitScope destroyed in a different thread than it was created in.") {
    break;
  }
  threadLocalEventLoop = nullptr;
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
void WaitScope::poll() {
  KJ_REQUIRE(&loop == threadLocalEventLoop, "WaitScope not valid for this thread.");
  KJ_REQUIRE(!loop.running, "poll() is not allowed from within event callbacks.");

  loop.running = true;
  KJ_DEFER(loop.running = false);

  for (;;) {
    if (!loop.turn()) {
      // No events in the queue.  Poll for I/O.
      loop.port.poll();

      if (!loop.isRunnable()) {
        // Still no events in the queue. We're done.
        return;
      }
    }
  }
}

425
namespace _ {  // private
426

427 428 429
void waitImpl(Own<_::PromiseNode>&& node, _::ExceptionOrValue& result, WaitScope& waitScope) {
  EventLoop& loop = waitScope.loop;
  KJ_REQUIRE(&loop == threadLocalEventLoop, "WaitScope not valid for this thread.");
430
  KJ_REQUIRE(!loop.running, "wait() is not allowed from within event callbacks.");
431

432
  BoolEvent doneEvent;
433
  node->setSelfPointer(&node);
434
  node->onReady(&doneEvent);
435 436 437 438 439 440 441 442 443

  loop.running = true;
  KJ_DEFER(loop.running = false);

  while (!doneEvent.fired) {
    if (!loop.turn()) {
      // No events in the queue.  Wait for callback.
      loop.port.wait();
    }
444
  }
445

446
  loop.setRunnable(loop.isRunnable());
447

448
  node->get(result);
449
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() {
450 451 452 453
    node = nullptr;
  })) {
    result.addException(kj::mv(*exception));
  }
454 455
}

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
bool pollImpl(_::PromiseNode& node, WaitScope& waitScope) {
  EventLoop& loop = waitScope.loop;
  KJ_REQUIRE(&loop == threadLocalEventLoop, "WaitScope not valid for this thread.");
  KJ_REQUIRE(!loop.running, "poll() is not allowed from within event callbacks.");

  BoolEvent doneEvent;
  node.onReady(&doneEvent);

  loop.running = true;
  KJ_DEFER(loop.running = false);

  while (!doneEvent.fired) {
    if (!loop.turn()) {
      // No events in the queue.  Poll for I/O.
      loop.port.poll();

      if (!doneEvent.fired && !loop.isRunnable()) {
        // No progress. Give up.
        node.onReady(nullptr);
        loop.setRunnable(false);
        return false;
      }
    }
  }

  loop.setRunnable(loop.isRunnable());
  return true;
}

485
Promise<void> yield() {
486 487 488
  return Promise<void>(false, kj::heap<YieldPromiseNode>());
}

489 490 491 492
Own<PromiseNode> neverDone() {
  return kj::heap<NeverDonePromiseNode>();
}

Kenton Varda's avatar
Kenton Varda committed
493
void NeverDone::wait(WaitScope& waitScope) const {
494 495 496 497 498
  ExceptionOr<Void> dummy;
  waitImpl(neverDone(), dummy, waitScope);
  KJ_UNREACHABLE;
}

499
void detach(kj::Promise<void>&& promise) {
500 501 502
  EventLoop& loop = currentEventLoop();
  KJ_REQUIRE(loop.daemons.get() != nullptr, "EventLoop is shutting down.") { return; }
  loop.daemons->add(kj::mv(promise));
503 504
}

505
Event::Event()
506
    : loop(currentEventLoop()), next(nullptr), prev(nullptr) {}
507

508
Event::~Event() noexcept(false) {
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
  if (prev != nullptr) {
    if (loop.tail == &next) {
      loop.tail = prev;
    }
    if (loop.depthFirstInsertPoint == &next) {
      loop.depthFirstInsertPoint = prev;
    }

    *prev = next;
    if (next != nullptr) {
      next->prev = prev;
    }
  }

  KJ_REQUIRE(!firing, "Promise callback destroyed itself.");
  KJ_REQUIRE(threadLocalEventLoop == &loop || threadLocalEventLoop == nullptr,
             "Promise destroyed from a different thread than it was created in.");
Kenton Varda's avatar
Kenton Varda committed
526 527
}

528
void Event::armDepthFirst() {
529 530 531
  KJ_REQUIRE(threadLocalEventLoop == &loop || threadLocalEventLoop == nullptr,
             "Event armed from different thread than it was created in.  You must use "
             "the thread-safe work queue to queue events cross-thread.");
532

533 534 535 536 537 538 539
  if (prev == nullptr) {
    next = *loop.depthFirstInsertPoint;
    prev = loop.depthFirstInsertPoint;
    *prev = this;
    if (next != nullptr) {
      next->prev = &next;
    }
540

541 542 543 544 545
    loop.depthFirstInsertPoint = &next;

    if (loop.tail == prev) {
      loop.tail = &next;
    }
546 547

    loop.setRunnable(true);
548 549 550
  }
}

551
void Event::armBreadthFirst() {
552 553 554 555 556 557 558 559 560 561 562 563 564
  KJ_REQUIRE(threadLocalEventLoop == &loop || threadLocalEventLoop == nullptr,
             "Event armed from different thread than it was created in.  You must use "
             "the thread-safe work queue to queue events cross-thread.");

  if (prev == nullptr) {
    next = *loop.tail;
    prev = loop.tail;
    *prev = this;
    if (next != nullptr) {
      next->prev = &next;
    }

    loop.tail = &next;
565 566

    loop.setRunnable(true);
567 568 569
  }
}

570
_::PromiseNode* Event::getInnerForTrace() {
571 572 573
  return nullptr;
}

574
#if !KJ_NO_RTTI
575 576 577 578
#if __GNUC__
static kj::String demangleTypeName(const char* name) {
  int status;
  char* buf = abi::__cxa_demangle(name, nullptr, nullptr, &status);
579
  kj::String result = kj::heapString(buf == nullptr ? name : buf);
580 581 582 583 584 585 586 587
  free(buf);
  return kj::mv(result);
}
#else
static kj::String demangleTypeName(const char* name) {
  return kj::heapString(name);
}
#endif
588
#endif
589

590
static kj::String traceImpl(Event* event, _::PromiseNode* node) {
591 592 593
#if KJ_NO_RTTI
  return heapString("Trace not available because RTTI is disabled.");
#else
594 595 596 597 598 599 600 601 602 603 604 605
  kj::Vector<kj::String> trace;

  if (event != nullptr) {
    trace.add(demangleTypeName(typeid(*event).name()));
  }

  while (node != nullptr) {
    trace.add(demangleTypeName(typeid(*node).name()));
    node = node->getInnerForTrace();
  }

  return strArray(trace, "\n");
606
#endif
607 608
}

609
kj::String Event::trace() {
610
  return traceImpl(this, getInnerForTrace());
611 612
}

613 614
}  // namespace _ (private)

615 616
// =======================================================================================

617 618
namespace _ {  // private

619 620 621 622
kj::String PromiseBase::trace() {
  return traceImpl(nullptr, node);
}

623 624
void PromiseNode::setSelfPointer(Own<PromiseNode>* selfPtr) noexcept {}

625
PromiseNode* PromiseNode::getInnerForTrace() { return nullptr; }
626

627
void PromiseNode::OnReadyEvent::init(Event* newEvent) {
628
  if (event == _kJ_ALREADY_READY) {
Kenton Varda's avatar
Kenton Varda committed
629 630 631
    // A new continuation was added to a promise that was already ready.  In this case, we schedule
    // breadth-first, to make it difficult for applications to accidentally starve the event loop
    // by repeatedly waiting on immediate promises.
632
    if (newEvent) newEvent->armBreadthFirst();
633
  } else {
634
    event = newEvent;
635 636 637
  }
}

638
void PromiseNode::OnReadyEvent::arm() {
639 640 641
  KJ_ASSERT(event != _kJ_ALREADY_READY, "arm() should only be called once");

  if (event != nullptr) {
Kenton Varda's avatar
Kenton Varda committed
642 643 644
    // A promise resolved and an event is already waiting on it.  In this case, arm in depth-first
    // order so that the event runs immediately after the current one.  This way, chained promises
    // execute together for better cache locality and lower latency.
645
    event->armDepthFirst();
646
  }
647 648

  event = _kJ_ALREADY_READY;
649 650
}

651 652
// -------------------------------------------------------------------

653 654 655
ImmediatePromiseNodeBase::ImmediatePromiseNodeBase() {}
ImmediatePromiseNodeBase::~ImmediatePromiseNodeBase() noexcept(false) {}

656 657
void ImmediatePromiseNodeBase::onReady(Event* event) noexcept {
  if (event) event->armBreadthFirst();
Kenton Varda's avatar
Kenton Varda committed
658
}
659 660 661 662 663 664 665 666

ImmediateBrokenPromiseNode::ImmediateBrokenPromiseNode(Exception&& exception)
    : exception(kj::mv(exception)) {}

void ImmediateBrokenPromiseNode::get(ExceptionOrValue& output) noexcept {
  output.exception = kj::mv(exception);
}

667 668
// -------------------------------------------------------------------

669 670 671 672
AttachmentPromiseNodeBase::AttachmentPromiseNodeBase(Own<PromiseNode>&& dependencyParam)
    : dependency(kj::mv(dependencyParam)) {
  dependency->setSelfPointer(&dependency);
}
673

674
void AttachmentPromiseNodeBase::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
675
  dependency->onReady(event);
676 677 678 679 680 681
}

void AttachmentPromiseNodeBase::get(ExceptionOrValue& output) noexcept {
  dependency->get(output);
}

682 683
PromiseNode* AttachmentPromiseNodeBase::getInnerForTrace() {
  return dependency;
684 685 686 687 688 689 690 691
}

void AttachmentPromiseNodeBase::dropDependency() {
  dependency = nullptr;
}

// -------------------------------------------------------------------

Kenton Varda's avatar
Kenton Varda committed
692 693 694
TransformPromiseNodeBase::TransformPromiseNodeBase(
    Own<PromiseNode>&& dependencyParam, void* continuationTracePtr)
    : dependency(kj::mv(dependencyParam)), continuationTracePtr(continuationTracePtr) {
695 696
  dependency->setSelfPointer(&dependency);
}
697

698
void TransformPromiseNodeBase::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
699
  dependency->onReady(event);
700 701 702 703 704
}

void TransformPromiseNodeBase::get(ExceptionOrValue& output) noexcept {
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() {
    getImpl(output);
705
    dropDependency();
706 707 708 709 710
  })) {
    output.addException(kj::mv(*exception));
  }
}

711 712
PromiseNode* TransformPromiseNodeBase::getInnerForTrace() {
  return dependency;
713 714
}

715 716 717 718
void TransformPromiseNodeBase::dropDependency() {
  dependency = nullptr;
}

719 720 721 722 723 724 725
void TransformPromiseNodeBase::getDepResult(ExceptionOrValue& output) {
  dependency->get(output);
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() {
    dependency = nullptr;
  })) {
    output.addException(kj::mv(*exception));
  }
Kenton Varda's avatar
Kenton Varda committed
726 727 728 729

  KJ_IF_MAYBE(e, output.exception) {
    e->addTrace(continuationTracePtr);
  }
730 731
}

732 733
// -------------------------------------------------------------------

734
ForkBranchBase::ForkBranchBase(Own<ForkHubBase>&& hubParam): hub(kj::mv(hubParam)) {
735
  if (hub->tailBranch == nullptr) {
736
    onReadyEvent.arm();
737 738
  } else {
    // Insert into hub's linked list of branches.
739
    prevPtr = hub->tailBranch;
740 741
    *prevPtr = this;
    next = nullptr;
742
    hub->tailBranch = &next;
743 744 745
  }
}

Kenton Varda's avatar
Kenton Varda committed
746
ForkBranchBase::~ForkBranchBase() noexcept(false) {
747 748 749
  if (prevPtr != nullptr) {
    // Remove from hub's linked list of branches.
    *prevPtr = next;
750
    (next == nullptr ? hub->tailBranch : next->prevPtr) = prevPtr;
751 752 753 754
  }
}

void ForkBranchBase::hubReady() noexcept {
755
  onReadyEvent.arm();
756 757 758 759
}

void ForkBranchBase::releaseHub(ExceptionOrValue& output) {
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([this]() {
760
    hub = nullptr;
761 762 763 764 765
  })) {
    output.addException(kj::mv(*exception));
  }
}

766
void ForkBranchBase::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
767
  onReadyEvent.init(event);
768 769
}

770 771
PromiseNode* ForkBranchBase::getInnerForTrace() {
  return hub->getInnerForTrace();
772 773 774 775
}

// -------------------------------------------------------------------

776 777
ForkHubBase::ForkHubBase(Own<PromiseNode>&& innerParam, ExceptionOrValue& resultRef)
    : inner(kj::mv(innerParam)), resultRef(resultRef) {
778
  inner->setSelfPointer(&inner);
779
  inner->onReady(this);
780
}
781

782
Maybe<Own<Event>> ForkHubBase::fire() {
783 784 785 786 787 788 789
  // Dependency is ready.  Fetch its result and then delete the node.
  inner->get(resultRef);
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([this]() {
    inner = nullptr;
  })) {
    resultRef.addException(kj::mv(*exception));
  }
790

791
  for (auto branch = headBranch; branch != nullptr; branch = branch->next) {
792 793 794
    branch->hubReady();
    *branch->prevPtr = nullptr;
    branch->prevPtr = nullptr;
795
  }
796
  *tailBranch = nullptr;
797 798

  // Indicate that the list is no longer active.
799
  tailBranch = nullptr;
800 801 802 803 804 805

  return nullptr;
}

_::PromiseNode* ForkHubBase::getInnerForTrace() {
  return inner;
806 807 808 809
}

// -------------------------------------------------------------------

810 811
ChainPromiseNode::ChainPromiseNode(Own<PromiseNode> innerParam)
    : state(STEP1), inner(kj::mv(innerParam)) {
812
  inner->setSelfPointer(&inner);
813
  inner->onReady(this);
814 815
}

816
ChainPromiseNode::~ChainPromiseNode() noexcept(false) {}
817

818
void ChainPromiseNode::onReady(Event* event) noexcept {
819 820
  switch (state) {
    case STEP1:
821
      onReadyEvent = event;
Kenton Varda's avatar
Kenton Varda committed
822
      return;
823
    case STEP2:
Kenton Varda's avatar
Kenton Varda committed
824 825
      inner->onReady(event);
      return;
826 827 828 829
  }
  KJ_UNREACHABLE;
}

830 831 832 833 834 835 836 837 838
void ChainPromiseNode::setSelfPointer(Own<PromiseNode>* selfPtr) noexcept {
  if (state == STEP2) {
    *selfPtr = kj::mv(inner);  // deletes this!
    selfPtr->get()->setSelfPointer(selfPtr);
  } else {
    this->selfPtr = selfPtr;
  }
}

839
void ChainPromiseNode::get(ExceptionOrValue& output) noexcept {
840
  KJ_REQUIRE(state == STEP2);
841 842 843
  return inner->get(output);
}

844 845
PromiseNode* ChainPromiseNode::getInnerForTrace() {
  return inner;
846 847
}

848
Maybe<Own<Event>> ChainPromiseNode::fire() {
849
  KJ_REQUIRE(state != STEP2);
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864

  static_assert(sizeof(Promise<int>) == sizeof(PromiseBase),
      "This code assumes Promise<T> does not add any new members to PromiseBase.");

  ExceptionOr<PromiseBase> intermediate;
  inner->get(intermediate);

  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([this]() {
    inner = nullptr;
  })) {
    intermediate.addException(kj::mv(*exception));
  }

  KJ_IF_MAYBE(exception, intermediate.exception) {
    // There is an exception.  If there is also a value, delete it.
865
    kj::runCatchingExceptions([&]() { intermediate.value = nullptr; });
866 867 868 869 870 871 872 873 874
    // Now set step2 to a rejected promise.
    inner = heap<ImmediateBrokenPromiseNode>(kj::mv(*exception));
  } else KJ_IF_MAYBE(value, intermediate.value) {
    // There is a value and no exception.  The value is itself a promise.  Adopt it as our
    // step2.
    inner = kj::mv(value->node);
  } else {
    // We can only get here if inner->get() returned neither an exception nor a
    // value, which never actually happens.
875
    KJ_FAIL_ASSERT("Inner node returned empty value.");
876 877 878
  }
  state = STEP2;

879 880 881 882 883 884
  if (selfPtr != nullptr) {
    // Hey, we can shorten the chain here.
    auto chain = selfPtr->downcast<ChainPromiseNode>();
    *selfPtr = kj::mv(inner);
    selfPtr->get()->setSelfPointer(selfPtr);
    if (onReadyEvent != nullptr) {
885
      selfPtr->get()->onReady(onReadyEvent);
886
    }
887

888 889 890 891 892
    // Return our self-pointer so that the caller takes care of deleting it.
    return Own<Event>(kj::mv(chain));
  } else {
    inner->setSelfPointer(&inner);
    if (onReadyEvent != nullptr) {
893
      inner->onReady(onReadyEvent);
894 895 896 897
    }

    return nullptr;
  }
898 899
}

900 901
// -------------------------------------------------------------------

902 903
ExclusiveJoinPromiseNode::ExclusiveJoinPromiseNode(Own<PromiseNode> left, Own<PromiseNode> right)
    : left(*this, kj::mv(left)), right(*this, kj::mv(right)) {}
904 905 906

ExclusiveJoinPromiseNode::~ExclusiveJoinPromiseNode() noexcept(false) {}

907
void ExclusiveJoinPromiseNode::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
908
  onReadyEvent.init(event);
909 910 911
}

void ExclusiveJoinPromiseNode::get(ExceptionOrValue& output) noexcept {
912
  KJ_REQUIRE(left.get(output) || right.get(output), "get() called before ready.");
913 914
}

915 916 917 918 919 920
PromiseNode* ExclusiveJoinPromiseNode::getInnerForTrace() {
  auto result = left.getInnerForTrace();
  if (result == nullptr) {
    result = right.getInnerForTrace();
  }
  return result;
921 922 923
}

ExclusiveJoinPromiseNode::Branch::Branch(
924 925
    ExclusiveJoinPromiseNode& joinNode, Own<PromiseNode> dependencyParam)
    : joinNode(joinNode), dependency(kj::mv(dependencyParam)) {
926
  dependency->setSelfPointer(&dependency);
927
  dependency->onReady(this);
928 929
}

930
ExclusiveJoinPromiseNode::Branch::~Branch() noexcept(false) {}
931 932

bool ExclusiveJoinPromiseNode::Branch::get(ExceptionOrValue& output) {
933
  if (dependency) {
934 935 936 937 938 939 940
    dependency->get(output);
    return true;
  } else {
    return false;
  }
}

941
Maybe<Own<Event>> ExclusiveJoinPromiseNode::Branch::fire() {
942 943 944
  // Cancel the branch that didn't return first.  Ignore exceptions caused by cancellation.
  if (this == &joinNode.left) {
    kj::runCatchingExceptions([&]() { joinNode.right.dependency = nullptr; });
945
  } else {
946 947
    kj::runCatchingExceptions([&]() { joinNode.left.dependency = nullptr; });
  }
948

949 950 951
  joinNode.onReadyEvent.arm();
  return nullptr;
}
952

953 954
PromiseNode* ExclusiveJoinPromiseNode::Branch::getInnerForTrace() {
  return dependency;
955 956 957 958
}

// -------------------------------------------------------------------

959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
ArrayJoinPromiseNodeBase::ArrayJoinPromiseNodeBase(
    Array<Own<PromiseNode>> promises, ExceptionOrValue* resultParts, size_t partSize)
    : countLeft(promises.size()) {
  // Make the branches.
  auto builder = heapArrayBuilder<Branch>(promises.size());
  for (uint i: indices(promises)) {
    ExceptionOrValue& output = *reinterpret_cast<ExceptionOrValue*>(
        reinterpret_cast<byte*>(resultParts) + i * partSize);
    builder.add(*this, kj::mv(promises[i]), output);
  }
  branches = builder.finish();

  if (branches.size() == 0) {
    onReadyEvent.arm();
  }
}
ArrayJoinPromiseNodeBase::~ArrayJoinPromiseNodeBase() noexcept(false) {}

977
void ArrayJoinPromiseNodeBase::onReady(Event* event) noexcept {
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
  onReadyEvent.init(event);
}

void ArrayJoinPromiseNodeBase::get(ExceptionOrValue& output) noexcept {
  // If any of the elements threw exceptions, propagate them.
  for (auto& branch: branches) {
    KJ_IF_MAYBE(exception, branch.getPart()) {
      output.addException(kj::mv(*exception));
    }
  }

  if (output.exception == nullptr) {
    // No errors.  The template subclass will need to fill in the result.
    getNoError(output);
  }
}

PromiseNode* ArrayJoinPromiseNodeBase::getInnerForTrace() {
  return branches.size() == 0 ? nullptr : branches[0].getInnerForTrace();
}

ArrayJoinPromiseNodeBase::Branch::Branch(
    ArrayJoinPromiseNodeBase& joinNode, Own<PromiseNode> dependencyParam, ExceptionOrValue& output)
    : joinNode(joinNode), dependency(kj::mv(dependencyParam)), output(output) {
  dependency->setSelfPointer(&dependency);
1003
  dependency->onReady(this);
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
}

ArrayJoinPromiseNodeBase::Branch::~Branch() noexcept(false) {}

Maybe<Own<Event>> ArrayJoinPromiseNodeBase::Branch::fire() {
  if (--joinNode.countLeft == 0) {
    joinNode.onReadyEvent.arm();
  }
  return nullptr;
}

_::PromiseNode* ArrayJoinPromiseNodeBase::Branch::getInnerForTrace() {
  return dependency->getInnerForTrace();
}

Maybe<Exception> ArrayJoinPromiseNodeBase::Branch::getPart() {
  dependency->get(output);
  return kj::mv(output.exception);
}

Kenton Varda's avatar
Kenton Varda committed
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
ArrayJoinPromiseNode<void>::ArrayJoinPromiseNode(
    Array<Own<PromiseNode>> promises, Array<ExceptionOr<_::Void>> resultParts)
    : ArrayJoinPromiseNodeBase(kj::mv(promises), resultParts.begin(), sizeof(ExceptionOr<_::Void>)),
      resultParts(kj::mv(resultParts)) {}

ArrayJoinPromiseNode<void>::~ArrayJoinPromiseNode() {}

void ArrayJoinPromiseNode<void>::getNoError(ExceptionOrValue& output) noexcept {
  output.as<_::Void>() = _::Void();
}

}  // namespace _ (private)

Promise<void> joinPromises(Array<Promise<void>>&& promises) {
  return Promise<void>(false, kj::heap<_::ArrayJoinPromiseNode<void>>(
      KJ_MAP(p, promises) { return kj::mv(p.node); },
      heapArray<_::ExceptionOr<_::Void>>(promises.size())));
}

namespace _ {  // (private)

1045 1046
// -------------------------------------------------------------------

1047 1048 1049
EagerPromiseNodeBase::EagerPromiseNodeBase(
    Own<PromiseNode>&& dependencyParam, ExceptionOrValue& resultRef)
    : dependency(kj::mv(dependencyParam)), resultRef(resultRef) {
1050
  dependency->setSelfPointer(&dependency);
1051
  dependency->onReady(this);
1052 1053
}

1054
void EagerPromiseNodeBase::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
1055
  onReadyEvent.init(event);
1056 1057
}

1058 1059
PromiseNode* EagerPromiseNodeBase::getInnerForTrace() {
  return dependency;
1060 1061
}

1062
Maybe<Own<Event>> EagerPromiseNodeBase::fire() {
1063 1064 1065 1066 1067
  dependency->get(resultRef);
  KJ_IF_MAYBE(exception, kj::runCatchingExceptions([this]() {
    dependency = nullptr;
  })) {
    resultRef.addException(kj::mv(*exception));
1068
  }
1069 1070 1071

  onReadyEvent.arm();
  return nullptr;
1072 1073
}

1074 1075
// -------------------------------------------------------------------

1076
void AdapterPromiseNodeBase::onReady(Event* event) noexcept {
Kenton Varda's avatar
Kenton Varda committed
1077
  onReadyEvent.init(event);
1078 1079
}

Kenton Varda's avatar
Kenton Varda committed
1080 1081 1082 1083
// -------------------------------------------------------------------

Promise<void> IdentityFunc<Promise<void>>::operator()() const { return READY_NOW; }

1084
}  // namespace _ (private)
1085
}  // namespace kj