[promises] Add always inline to basic combinators (#36981)

These are substitutes for language features and never warrant a new function being created. Significantly boosts the benchmarks in #36946

Closes #36981

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36981 from ctiller:seqqy 1a075be7e9
PiperOrigin-RevId: 645169296

src/core/lib/promise/all_ok.h

@@ -39,17 +39,21 @@ struct AllOkTraits {
   template <typename T>
   using ResultType = Result;
   template <typename T>
-  static bool IsOk(const T& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T& x) {
     return IsStatusOk(x);
   }
-  static Empty Unwrapped(StatusFlag) { return Empty{}; }
-  static Empty Unwrapped(absl::Status) { return Empty{}; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(StatusFlag) {
+    return Empty{};
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(absl::Status) {
+    return Empty{};
+  }
   template <typename R, typename T>
-  static R EarlyReturn(T&& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(T&& x) {
     return StatusCast<R>(std::forward<T>(x));
   }
   template <typename... A>
-  static Result FinalReturn(A&&...) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Result FinalReturn(A&&...) {
     return Result{};
   }
 };
@@ -58,8 +62,11 @@ struct AllOkTraits {
 template <typename Result, typename... Promises>
 class AllOk {
  public:
-  explicit AllOk(Promises... promises) : state_(std::move(promises)...) {}
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit AllOk(Promises... promises)
+      : state_(std::move(promises)...) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   JoinState<AllOkTraits<Result>, Promises...> state_;
@@ -71,7 +78,7 @@ class AllOk {
 // If any fail, cancel the rest and return the failure.
 // If all succeed, return Ok.
 template <typename Result, typename... Promises>
-auto AllOk(Promises... promises) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto AllOk(Promises... promises) {
   return promise_detail::AllOk<Result, Promises...>(std::move(promises)...);
 }
 

src/core/lib/promise/cancel_callback.h

@@ -28,26 +28,28 @@ namespace cancel_callback_detail {
 template <typename Fn>
 class Handler {
  public:
-  explicit Handler(Fn fn) : fn_(std::move(fn)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Handler(Fn fn)
+      : fn_(std::move(fn)) {}
   Handler(const Handler&) = delete;
   Handler& operator=(const Handler&) = delete;
-  ~Handler() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Handler() {
     if (!done_) {
       promise_detail::Context<Arena> ctx(arena_.get());
       fn_();
     }
   }
-  Handler(Handler&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Handler(Handler&& other) noexcept
       : fn_(std::move(other.fn_)), done_(other.done_) {
     other.done_ = true;
   }
-  Handler& operator=(Handler&& other) noexcept {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Handler& operator=(
+      Handler&& other) noexcept {
     fn_ = std::move(other.fn_);
     done_ = other.done_;
     other.done_ = true;
   }
 
-  void Done() { done_ = true; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void Done() { done_ = true; }
 
  private:
   Fn fn_;
@@ -67,7 +69,8 @@ class Handler {
 // completion.
 // Returns a promise with the same result type as main_fn.
 template <typename MainFn, typename CancelFn>
-auto OnCancel(MainFn main_fn, CancelFn cancel_fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto OnCancel(MainFn main_fn,
+                                                   CancelFn cancel_fn) {
   return [on_cancel =
               cancel_callback_detail::Handler<CancelFn>(std::move(cancel_fn)),
           main_fn = promise_detail::PromiseLike<MainFn>(
@@ -84,7 +87,8 @@ auto OnCancel(MainFn main_fn, CancelFn cancel_fn) {
 // resulting promise. If the factory is dropped without being called, cancel_fn
 // is called.
 template <typename MainFn, typename CancelFn>
-auto OnCancelFactory(MainFn main_fn, CancelFn cancel_fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto OnCancelFactory(MainFn main_fn,
+                                                          CancelFn cancel_fn) {
   return [on_cancel =
               cancel_callback_detail::Handler<CancelFn>(std::move(cancel_fn)),
           main_fn = std::move(main_fn)]() mutable {

src/core/lib/promise/detail/promise_factory.h

@@ -115,41 +115,44 @@ class Curried {
 // Promote a callable(A) -> T | Poll<T> to a PromiseFactory(A) -> Promise<T> by
 // capturing A.
 template <typename A, typename F>
-absl::enable_if_t<!IsVoidCallable<ResultOf<F(A)>>::value,
-                  PromiseLike<Curried<RemoveCVRef<F>, A>>>
-PromiseFactoryImpl(F&& f, A&& arg) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    absl::enable_if_t<!IsVoidCallable<ResultOf<F(A)>>::value,
+                      PromiseLike<Curried<RemoveCVRef<F>, A>>>
+    PromiseFactoryImpl(F&& f, A&& arg) {
   return Curried<RemoveCVRef<F>, A>(std::forward<F>(f), std::forward<A>(arg));
 }
 
 // Promote a callable() -> T|Poll<T> to a PromiseFactory(A) -> Promise<T>
 // by dropping the argument passed to the factory.
 template <typename A, typename F>
-absl::enable_if_t<!IsVoidCallable<ResultOf<F()>>::value,
-                  PromiseLike<RemoveCVRef<F>>>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::enable_if_t<
+    !IsVoidCallable<ResultOf<F()>>::value, PromiseLike<RemoveCVRef<F>>>
 PromiseFactoryImpl(F f, A&&) {
   return PromiseLike<F>(std::move(f));
 }
 
 // Promote a callable() -> T|Poll<T> to a PromiseFactory() -> Promise<T>
 template <typename F>
-absl::enable_if_t<!IsVoidCallable<ResultOf<F()>>::value,
-                  PromiseLike<RemoveCVRef<F>>>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::enable_if_t<
+    !IsVoidCallable<ResultOf<F()>>::value, PromiseLike<RemoveCVRef<F>>>
 PromiseFactoryImpl(F f) {
   return PromiseLike<F>(std::move(f));
 }
 
 // Given a callable(A) -> Promise<T>, name it a PromiseFactory and use it.
 template <typename A, typename F>
-absl::enable_if_t<IsVoidCallable<ResultOf<F(A)>>::value,
-                  PromiseLike<decltype(std::declval<F>()(std::declval<A>()))>>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::enable_if_t<
+    IsVoidCallable<ResultOf<F(A)>>::value,
+    PromiseLike<decltype(std::declval<F>()(std::declval<A>()))>>
 PromiseFactoryImpl(F&& f, A&& arg) {
   return f(std::forward<A>(arg));
 }
 
 // Given a callable(A) -> Promise<T>, name it a PromiseFactory and use it.
 template <typename A, typename F>
-absl::enable_if_t<IsVoidCallable<ResultOf<F(A)>>::value,
-                  PromiseLike<decltype(std::declval<F>()(std::declval<A>()))>>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::enable_if_t<
+    IsVoidCallable<ResultOf<F(A)>>::value,
+    PromiseLike<decltype(std::declval<F>()(std::declval<A>()))>>
 PromiseFactoryImpl(F& f, A&& arg) {
   return f(std::forward<A>(arg));
 }
@@ -157,17 +160,19 @@ PromiseFactoryImpl(F& f, A&& arg) {
 // Given a callable() -> Promise<T>, promote it to a
 // PromiseFactory(A) -> Promise<T> by dropping the first argument.
 template <typename A, typename F>
-absl::enable_if_t<IsVoidCallable<ResultOf<F()>>::value,
-                  PromiseLike<decltype(std::declval<F>()())>>
-PromiseFactoryImpl(F&& f, A&&) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    absl::enable_if_t<IsVoidCallable<ResultOf<F()>>::value,
+                      PromiseLike<decltype(std::declval<F>()())>>
+    PromiseFactoryImpl(F&& f, A&&) {
   return f();
 }
 
 // Given a callable() -> Promise<T>, name it a PromiseFactory and use it.
 template <typename F>
-absl::enable_if_t<IsVoidCallable<ResultOf<F()>>::value,
-                  PromiseLike<decltype(std::declval<F>()())>>
-PromiseFactoryImpl(F&& f) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    absl::enable_if_t<IsVoidCallable<ResultOf<F()>>::value,
+                      PromiseLike<decltype(std::declval<F>()())>>
+    PromiseFactoryImpl(F&& f) {
   return f();
 }
 
@@ -181,9 +186,10 @@ class OncePromiseFactory {
   using Promise =
       decltype(PromiseFactoryImpl(std::move(f_), std::declval<A>()));
 
-  explicit OncePromiseFactory(F f) : f_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit OncePromiseFactory(F f)
+      : f_(std::move(f)) {}
 
-  Promise Make(Arg&& a) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make(Arg&& a) {
     return PromiseFactoryImpl(std::move(f_), std::forward<Arg>(a));
   }
 };
@@ -197,9 +203,12 @@ class OncePromiseFactory<void, F> {
   using Arg = void;
   using Promise = decltype(PromiseFactoryImpl(std::move(f_)));
 
-  explicit OncePromiseFactory(F f) : f_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit OncePromiseFactory(F f)
+      : f_(std::move(f)) {}
 
-  Promise Make() { return PromiseFactoryImpl(std::move(f_)); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make() {
+    return PromiseFactoryImpl(std::move(f_));
+  }
 };
 
 template <typename A, typename F>
@@ -211,12 +220,15 @@ class RepeatedPromiseFactory {
   using Arg = A;
   using Promise = decltype(PromiseFactoryImpl(f_, std::declval<A>()));
 
-  explicit RepeatedPromiseFactory(F f) : f_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit RepeatedPromiseFactory(F f)
+      : f_(std::move(f)) {}
 
-  Promise Make(Arg&& a) const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make(Arg&& a) const {
+    return PromiseFactoryImpl(f_, std::forward<Arg>(a));
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make(Arg&& a) {
     return PromiseFactoryImpl(f_, std::forward<Arg>(a));
   }
-  Promise Make(Arg&& a) { return PromiseFactoryImpl(f_, std::forward<Arg>(a)); }
 };
 
 template <typename F>
@@ -228,10 +240,15 @@ class RepeatedPromiseFactory<void, F> {
   using Arg = void;
   using Promise = decltype(PromiseFactoryImpl(f_));
 
-  explicit RepeatedPromiseFactory(F f) : f_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit RepeatedPromiseFactory(F f)
+      : f_(std::move(f)) {}
 
-  Promise Make() const { return PromiseFactoryImpl(f_); }
-  Promise Make() { return PromiseFactoryImpl(f_); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make() const {
+    return PromiseFactoryImpl(f_);
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Promise Make() {
+    return PromiseFactoryImpl(f_);
+  }
 };
 
 }  // namespace promise_detail

src/core/lib/promise/detail/promise_like.h

@@ -51,16 +51,21 @@ namespace promise_detail {
 
 template <typename T>
 struct PollWrapper {
-  static Poll<T> Wrap(T&& x) { return Poll<T>(std::forward<T>(x)); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Wrap(T&& x) {
+    return Poll<T>(std::forward<T>(x));
+  }
 };
 
 template <typename T>
 struct PollWrapper<Poll<T>> {
-  static Poll<T> Wrap(Poll<T>&& x) { return std::forward<Poll<T>>(x); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Wrap(Poll<T>&& x) {
+    return std::forward<Poll<T>>(x);
+  }
 };
 
 template <typename T>
-auto WrapInPoll(T&& x) -> decltype(PollWrapper<T>::Wrap(std::forward<T>(x))) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto WrapInPoll(T&& x)
+    -> decltype(PollWrapper<T>::Wrap(std::forward<T>(x))) {
   return PollWrapper<T>::Wrap(std::forward<T>(x));
 }
 
@@ -88,8 +93,12 @@ class PromiseLike<F, absl::enable_if_t<!std::is_void<
 
  public:
   // NOLINTNEXTLINE - internal detail that drastically simplifies calling code.
-  PromiseLike(F&& f) : f_(std::forward<F>(f)) {}
-  auto operator()() -> decltype(WrapInPoll(f_())) { return WrapInPoll(f_()); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION PromiseLike(F&& f)
+      : f_(std::forward<F>(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()()
+      -> decltype(WrapInPoll(f_())) {
+    return WrapInPoll(f_());
+  }
   using Result = typename PollTraits<decltype(WrapInPoll(f_()))>::Type;
 };
 

src/core/lib/promise/detail/seq_state.h

@@ -102,11 +102,13 @@ struct SeqState<Traits, P, F0> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, DebugLocation whence) noexcept : whence(whence) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0,
+                                                DebugLocation whence) noexcept
+      : whence(whence) {
     Construct(&prior.current_promise, std::forward<P>(p));
     Construct(&prior.next_factory, std::forward<F0>(f0));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.current_promise);
@@ -118,14 +120,14 @@ struct SeqState<Traits, P, F0> {
   tail0:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -139,8 +141,9 @@ struct SeqState<Traits, P, F0> {
   tail0:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -230,13 +233,14 @@ struct SeqState<Traits, P, F0, F1> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.current_promise, std::forward<P>(p));
     Construct(&prior.prior.next_factory, std::forward<F0>(f0));
     Construct(&prior.next_factory, std::forward<F1>(f1));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.current_promise);
@@ -253,7 +257,7 @@ struct SeqState<Traits, P, F0, F1> {
   tail1:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -261,7 +265,7 @@ struct SeqState<Traits, P, F0, F1> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -282,8 +286,9 @@ struct SeqState<Traits, P, F0, F1> {
   tail1:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -417,14 +422,16 @@ struct SeqState<Traits, P, F0, F1, F2> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.current_promise, std::forward<P>(p));
     Construct(&prior.prior.prior.next_factory, std::forward<F0>(f0));
     Construct(&prior.prior.next_factory, std::forward<F1>(f1));
     Construct(&prior.next_factory, std::forward<F2>(f2));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.current_promise);
@@ -446,7 +453,7 @@ struct SeqState<Traits, P, F0, F1, F2> {
   tail2:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -456,7 +463,7 @@ struct SeqState<Traits, P, F0, F1, F2> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -484,8 +491,9 @@ struct SeqState<Traits, P, F0, F1, F2> {
   tail2:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -663,8 +671,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3,
-           DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.current_promise, std::forward<P>(p));
     Construct(&prior.prior.prior.prior.next_factory, std::forward<F0>(f0));
@@ -672,7 +681,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3> {
     Construct(&prior.prior.next_factory, std::forward<F2>(f2));
     Construct(&prior.next_factory, std::forward<F3>(f3));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.current_promise);
@@ -699,7 +708,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3> {
   tail3:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -711,7 +720,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -746,8 +755,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3> {
   tail3:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -976,8 +986,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4,
-           DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.current_promise,
               std::forward<P>(p));
@@ -988,7 +999,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4> {
     Construct(&prior.prior.next_factory, std::forward<F3>(f3));
     Construct(&prior.next_factory, std::forward<F4>(f4));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.current_promise);
@@ -1020,7 +1031,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4> {
   tail4:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -1034,7 +1045,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -1077,8 +1088,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4> {
   tail4:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -1353,8 +1365,10 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5,
-           DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.current_promise,
               std::forward<P>(p));
@@ -1367,7 +1381,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5> {
     Construct(&prior.prior.next_factory, std::forward<F4>(f4));
     Construct(&prior.next_factory, std::forward<F5>(f5));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.current_promise);
@@ -1404,7 +1418,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5> {
   tail5:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -1420,7 +1434,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -1473,8 +1487,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5> {
   tail5:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -1796,8 +1811,10 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.prior.current_promise,
               std::forward<P>(p));
@@ -1812,7 +1829,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6> {
     Construct(&prior.prior.next_factory, std::forward<F5>(f5));
     Construct(&prior.next_factory, std::forward<F6>(f6));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.prior.current_promise);
@@ -1854,7 +1871,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6> {
   tail6:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -1872,7 +1889,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -1935,8 +1952,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6> {
   tail6:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -2306,8 +2324,10 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           F7&& f7, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6, F7&& f7,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.prior.prior.current_promise,
               std::forward<P>(p));
@@ -2324,7 +2344,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7> {
     Construct(&prior.prior.next_factory, std::forward<F6>(f6));
     Construct(&prior.next_factory, std::forward<F7>(f7));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(
@@ -2372,7 +2392,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7> {
   tail7:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -2393,7 +2413,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -2466,8 +2486,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7> {
   tail7:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -2886,8 +2907,11 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           F7&& f7, F8&& f8, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6, F7&& f7,
+                                                F8&& f8,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(
         &prior.prior.prior.prior.prior.prior.prior.prior.prior.current_promise,
@@ -2908,7 +2932,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8> {
     Construct(&prior.prior.next_factory, std::forward<F7>(f7));
     Construct(&prior.next_factory, std::forward<F8>(f8));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.prior.prior.prior
@@ -2963,7 +2987,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8> {
   tail8:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -2988,7 +3012,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -3072,8 +3096,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8> {
   tail8:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -3543,8 +3568,11 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           F7&& f7, F8&& f8, F9&& f9, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6, F7&& f7,
+                                                F8&& f8, F9&& f9,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
                    .current_promise,
@@ -3568,7 +3596,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> {
     Construct(&prior.prior.next_factory, std::forward<F8>(f8));
     Construct(&prior.next_factory, std::forward<F9>(f9));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
@@ -3630,7 +3658,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> {
   tail9:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -3659,7 +3687,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -3754,8 +3782,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> {
   tail9:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -4277,8 +4306,11 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           F7&& f7, F8&& f8, F9&& f9, F10&& f10, DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6, F7&& f7,
+                                                F8&& f8, F9&& f9, F10&& f10,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
                    .current_promise,
@@ -4305,7 +4337,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> {
     Construct(&prior.prior.next_factory, std::forward<F9>(f9));
     Construct(&prior.next_factory, std::forward<F10>(f10));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
@@ -4374,7 +4406,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> {
   tail10:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -4407,7 +4439,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -4513,8 +4545,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> {
   tail10:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
@@ -5089,9 +5122,12 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p, F0&& f0, F1&& f1, F2&& f2, F3&& f3, F4&& f4, F5&& f5, F6&& f6,
-           F7&& f7, F8&& f8, F9&& f9, F10&& f10, F11&& f11,
-           DebugLocation whence) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p, F0&& f0, F1&& f1,
+                                                F2&& f2, F3&& f3, F4&& f4,
+                                                F5&& f5, F6&& f6, F7&& f7,
+                                                F8&& f8, F9&& f9, F10&& f10,
+                                                F11&& f11,
+                                                DebugLocation whence) noexcept
       : whence(whence) {
     Construct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
                    .prior.current_promise,
@@ -5121,7 +5157,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> {
     Construct(&prior.prior.next_factory, std::forward<F10>(f10));
     Construct(&prior.next_factory, std::forward<F11>(f11));
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
       case State::kState0:
         Destruct(&prior.prior.prior.prior.prior.prior.prior.prior.prior.prior
@@ -5197,7 +5233,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> {
   tail11:
     Destruct(&prior.next_factory);
   }
-  SeqState(const SeqState& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept
       : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&prior.current_promise, other.prior.current_promise);
@@ -5234,7 +5270,7 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> {
     Construct(&prior.next_factory, other.prior.next_factory);
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept
       : state(other.state), whence(other.whence) {
     switch (state) {
       case State::kState0:
@@ -5352,8 +5388,9 @@ struct SeqState<Traits, P, F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> {
   tail11:
     Construct(&prior.next_factory, std::move(other.prior.next_factory));
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) =
+      delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
       case State::kState0: {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {

src/core/lib/promise/detail/status.h

@@ -30,12 +30,14 @@ namespace promise_detail {
 
 // Convert with a move the input status to an absl::Status.
 template <typename T>
-absl::Status IntoStatus(absl::StatusOr<T>* status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::Status IntoStatus(
+    absl::StatusOr<T>* status) {
   return std::move(status->status());
 }
 
 // Convert with a move the input status to an absl::Status.
-inline absl::Status IntoStatus(absl::Status* status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline absl::Status IntoStatus(
+    absl::Status* status) {
   return std::move(*status);
 }
 
@@ -44,10 +46,14 @@ inline absl::Status IntoStatus(absl::Status* status) {
 // Return true if the status represented by the argument is ok, false if not.
 // By implementing this function for other, non-absl::Status types, those types
 // can participate in TrySeq as result types that affect control flow.
-inline bool IsStatusOk(const absl::Status& status) { return status.ok(); }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const absl::Status& status) {
+  return status.ok();
+}
 
 template <typename T>
-inline bool IsStatusOk(const absl::StatusOr<T>& status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const absl::StatusOr<T>& status) {
   return status.ok();
 }
 
@@ -56,29 +62,38 @@ struct StatusCastImpl;
 
 template <typename To>
 struct StatusCastImpl<To, To> {
-  static To Cast(To&& t) { return std::move(t); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static To Cast(To&& t) {
+    return std::move(t);
+  }
 };
 
 template <typename To>
 struct StatusCastImpl<To, const To&> {
-  static To Cast(const To& t) { return t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static To Cast(const To& t) { return t; }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, absl::StatusOr<T>> {
-  static absl::Status Cast(absl::StatusOr<T>&& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      absl::StatusOr<T>&& t) {
     return std::move(t.status());
   }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, absl::StatusOr<T>&> {
-  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      const absl::StatusOr<T>& t) {
+    return t.status();
+  }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, const absl::StatusOr<T>&> {
-  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      const absl::StatusOr<T>& t) {
+    return t.status();
+  }
 };
 
 // StatusCast<> allows casting from one status-bearing type to another,
@@ -88,7 +103,7 @@ struct StatusCastImpl<absl::Status, const absl::StatusOr<T>&> {
 // For cases where the status is guaranteed to be a failure (and hence not
 // needing to preserve values) see FailureStatusCast<> below.
 template <typename To, typename From>
-To StatusCast(From&& from) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION To StatusCast(From&& from) {
   return StatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }
 
@@ -97,16 +112,22 @@ struct FailureStatusCastImpl : public StatusCastImpl<To, From> {};
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, absl::Status> {
-  static absl::StatusOr<T> Cast(absl::Status&& t) { return std::move(t); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      absl::Status&& t) {
+    return std::move(t);
+  }
 };
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, const absl::Status&> {
-  static absl::StatusOr<T> Cast(const absl::Status& t) { return t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      const absl::Status& t) {
+    return t;
+  }
 };
 
 template <typename To, typename From>
-To FailureStatusCast(From&& from) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION To FailureStatusCast(From&& from) {
   DCHECK(!IsStatusOk(from));
   return FailureStatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }

src/core/lib/promise/for_each.h

@@ -47,14 +47,17 @@ struct Done;
 
 template <>
 struct Done<absl::Status> {
-  static absl::Status Make(bool cancelled) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Make(
+      bool cancelled) {
     return cancelled ? absl::CancelledError() : absl::OkStatus();
   }
 };
 
 template <>
 struct Done<StatusFlag> {
-  static StatusFlag Make(bool cancelled) { return StatusFlag(!cancelled); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static StatusFlag Make(bool cancelled) {
+    return StatusFlag(!cancelled);
+  }
 };
 
 template <typename T, typename SfinaeVoid = void>
@@ -70,20 +73,23 @@ template <typename T>
 struct NextValueTraits<T, absl::void_t<typename T::value_type>> {
   using Value = typename T::value_type;
 
-  static NextValueType Type(const T& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static NextValueType Type(const T& t) {
     if (t.has_value()) return NextValueType::kValue;
     if (t.cancelled()) return NextValueType::kError;
     return NextValueType::kEndOfStream;
   }
 
-  static Value& MutableValue(T& t) { return *t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Value& MutableValue(T& t) {
+    return *t;
+  }
 };
 
 template <typename T>
 struct NextValueTraits<ValueOrFailure<absl::optional<T>>> {
   using Value = T;
 
-  static NextValueType Type(const ValueOrFailure<absl::optional<T>>& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static NextValueType Type(
+      const ValueOrFailure<absl::optional<T>>& t) {
     if (t.ok()) {
       if (t.value().has_value()) return NextValueType::kValue;
       return NextValueType::kEndOfStream;
@@ -91,7 +97,8 @@ struct NextValueTraits<ValueOrFailure<absl::optional<T>>> {
     return NextValueType::kError;
   }
 
-  static Value& MutableValue(ValueOrFailure<absl::optional<T>>& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Value& MutableValue(
+      ValueOrFailure<absl::optional<T>>& t) {
     return **t;
   }
 };
@@ -110,13 +117,14 @@ class ForEach {
  public:
   using Result =
       typename PollTraits<decltype(std::declval<ActionPromise>()())>::Type;
-  ForEach(Reader reader, Action action, DebugLocation whence = {})
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ForEach(Reader reader, Action action,
+                                               DebugLocation whence = {})
       : reader_(std::move(reader)),
         action_factory_(std::move(action)),
         whence_(whence) {
     Construct(&reader_next_, reader_.Next());
   }
-  ~ForEach() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~ForEach() {
     if (reading_next_) {
       Destruct(&reader_next_);
     } else {
@@ -163,7 +171,7 @@ class ForEach {
                         ":", whence_.line(), "]: ");
   }
 
-  Poll<Result> PollReaderNext() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollReaderNext() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
       gpr_log(GPR_INFO, "%s PollReaderNext", DebugTag().c_str());
     }
@@ -233,8 +241,8 @@ class ForEach {
 
 /// For each item acquired by calling Reader::Next, run the promise Action.
 template <typename Reader, typename Action>
-for_each_detail::ForEach<Reader, Action> ForEach(Reader reader, Action action,
-                                                 DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION for_each_detail::ForEach<Reader, Action>
+ForEach(Reader reader, Action action, DebugLocation whence = {}) {
   return for_each_detail::ForEach<Reader, Action>(std::move(reader),
                                                   std::move(action), whence);
 }

src/core/lib/promise/if.h

@@ -33,8 +33,8 @@ namespace grpc_core {
 namespace promise_detail {
 
 template <typename CallPoll, typename T, typename F>
-typename CallPoll::PollResult ChooseIf(CallPoll call_poll, bool result,
-                                       T* if_true, F* if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION typename CallPoll::PollResult ChooseIf(
+    CallPoll call_poll, bool result, T* if_true, F* if_false) {
   if (result) {
     auto promise = if_true->Make();
     return call_poll(promise);
@@ -45,9 +45,8 @@ typename CallPoll::PollResult ChooseIf(CallPoll call_poll, bool result,
 }
 
 template <typename CallPoll, typename T, typename F>
-typename CallPoll::PollResult ChooseIf(CallPoll call_poll,
-                                       absl::StatusOr<bool> result, T* if_true,
-                                       F* if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION typename CallPoll::PollResult ChooseIf(
+    CallPoll call_poll, absl::StatusOr<bool> result, T* if_true, F* if_false) {
   if (!result.ok()) {
     return typename CallPoll::PollResult(result.status());
   } else if (*result) {
@@ -71,12 +70,12 @@ class If {
       typename PollTraits<decltype(std::declval<TruePromise>()())>::Type;
 
  public:
-  If(C condition, T if_true, F if_false)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(C condition, T if_true, F if_false)
       : state_(Evaluating{ConditionPromise(std::move(condition)),
                           TrueFactory(std::move(if_true)),
                           FalseFactory(std::move(if_false))}) {}
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     return absl::visit(CallPoll<false>{this}, state_);
   }
 
@@ -95,7 +94,8 @@ class If {
 
     If* const self;
 
-    PollResult operator()(Evaluating& evaluating) const {
+    GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION PollResult
+    operator()(Evaluating& evaluating) const {
       static_assert(
           !kSetState,
           "shouldn't need to set state coming through the initial branch");
@@ -108,7 +108,8 @@ class If {
     }
 
     template <class Promise>
-    PollResult operator()(Promise& promise) const {
+    GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION PollResult
+    operator()(Promise& promise) const {
       auto r = promise();
       if (kSetState && r.pending()) {
         self->state_.template emplace<Promise>(std::move(promise));
@@ -129,7 +130,8 @@ class If<bool, T, F> {
       typename PollTraits<decltype(std::declval<TruePromise>()())>::Type;
 
  public:
-  If(bool condition, T if_true, F if_false) : condition_(condition) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(bool condition, T if_true, F if_false)
+      : condition_(condition) {
     TrueFactory true_factory(std::move(if_true));
     FalseFactory false_factory(std::move(if_false));
     if (condition_) {
@@ -138,7 +140,7 @@ class If<bool, T, F> {
       Construct(&if_false_, false_factory.Make());
     }
   }
-  ~If() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~If() {
     if (condition_) {
       Destruct(&if_true_);
     } else {
@@ -148,21 +150,22 @@ class If<bool, T, F> {
 
   If(const If&) = delete;
   If& operator=(const If&) = delete;
-  If(If&& other) noexcept : condition_(other.condition_) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(If&& other) noexcept
+      : condition_(other.condition_) {
     if (condition_) {
       Construct(&if_true_, std::move(other.if_true_));
     } else {
       Construct(&if_false_, std::move(other.if_false_));
     }
   }
-  If& operator=(If&& other) noexcept {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If& operator=(If&& other) noexcept {
     if (&other == this) return *this;
     Destruct(this);
     Construct(this, std::move(other));
     return *this;
   }
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
 #ifndef NDEBUG
     asan_canary_ = std::make_unique<int>(1 + *asan_canary_);
 #endif
@@ -197,7 +200,8 @@ class If<bool, T, F> {
 // This makes it safe to capture lambda arguments in the promise factory by
 // reference.
 template <typename C, typename T, typename F>
-promise_detail::If<C, T, F> If(C condition, T if_true, F if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::If<C, T, F> If(
+    C condition, T if_true, F if_false) {
   return promise_detail::If<C, T, F>(std::move(condition), std::move(if_true),
                                      std::move(if_false));
 }

src/core/lib/promise/join.h

@@ -33,11 +33,11 @@ struct JoinTraits {
   template <typename T>
   using ResultType = absl::remove_reference_t<T>;
   template <typename T>
-  static bool IsOk(const T&) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T&) {
     return true;
   }
   template <typename T>
-  static T Unwrapped(T x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T Unwrapped(T x) {
     return x;
   }
   template <typename R, typename T>
@@ -45,7 +45,8 @@ struct JoinTraits {
     abort();
   }
   template <typename... A>
-  static std::tuple<A...> FinalReturn(A... a) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static std::tuple<A...> FinalReturn(
+      A... a) {
     return std::make_tuple(std::move(a)...);
   }
 };
@@ -53,8 +54,11 @@ struct JoinTraits {
 template <typename... Promises>
 class Join {
  public:
-  explicit Join(Promises... promises) : state_(std::move(promises)...) {}
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Join(Promises... promises)
+      : state_(std::move(promises)...) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   JoinState<JoinTraits, Promises...> state_;
@@ -62,7 +66,7 @@ class Join {
 
 struct WrapInTuple {
   template <typename T>
-  std::tuple<T> operator()(T x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION std::tuple<T> operator()(T x) {
     return std::make_tuple(std::move(x));
   }
 };
@@ -72,12 +76,13 @@ struct WrapInTuple {
 /// Combinator to run all promises to completion, and return a tuple
 /// of their results.
 template <typename... Promise>
-promise_detail::Join<Promise...> Join(Promise... promises) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Join<Promise...> Join(
+    Promise... promises) {
   return promise_detail::Join<Promise...>(std::move(promises)...);
 }
 
 template <typename F>
-auto Join(F promise) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto Join(F promise) {
   return Map(std::move(promise), promise_detail::WrapInTuple{});
 }
 

src/core/lib/promise/loop.h

@@ -46,13 +46,17 @@ struct LoopTraits;
 template <typename T>
 struct LoopTraits<LoopCtl<T>> {
   using Result = T;
-  static LoopCtl<T> ToLoopCtl(LoopCtl<T> value) { return value; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<T> ToLoopCtl(
+      LoopCtl<T> value) {
+    return value;
+  }
 };
 
 template <typename T>
 struct LoopTraits<absl::StatusOr<LoopCtl<T>>> {
   using Result = absl::StatusOr<T>;
-  static LoopCtl<Result> ToLoopCtl(absl::StatusOr<LoopCtl<T>> value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<Result> ToLoopCtl(
+      absl::StatusOr<LoopCtl<T>> value) {
     if (!value.ok()) return value.status();
     auto& inner = *value;
     if (absl::holds_alternative<Continue>(inner)) return Continue{};
@@ -63,7 +67,7 @@ struct LoopTraits<absl::StatusOr<LoopCtl<T>>> {
 template <>
 struct LoopTraits<absl::StatusOr<LoopCtl<absl::Status>>> {
   using Result = absl::Status;
-  static LoopCtl<Result> ToLoopCtl(
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<Result> ToLoopCtl(
       absl::StatusOr<LoopCtl<absl::Status>> value) {
     if (!value.ok()) return value.status();
     const auto& inner = *value;
@@ -82,12 +86,13 @@ class Loop {
  public:
   using Result = typename LoopTraits<PromiseResult>::Result;
 
-  explicit Loop(F f) : factory_(std::move(f)) {}
-  ~Loop() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Loop(F f)
+      : factory_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Loop() {
     if (started_) Destruct(&promise_);
   }
 
-  Loop(Loop&& loop) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Loop(Loop&& loop) noexcept
       : factory_(std::move(loop.factory_)), started_(loop.started_) {
     if (started_) Construct(&promise_, std::move(loop.promise_));
   }
@@ -95,7 +100,7 @@ class Loop {
   Loop(const Loop& loop) = delete;
   Loop& operator=(const Loop& loop) = delete;
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     if (!started_) {
       started_ = true;
       Construct(&promise_, factory_.Make());
@@ -136,7 +141,7 @@ class Loop {
 // Expects F returns LoopCtl<T> - if it's Continue, then run the loop again -
 // otherwise yield the returned value as the result of the loop.
 template <typename F>
-promise_detail::Loop<F> Loop(F f) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Loop<F> Loop(F f) {
   return promise_detail::Loop<F>(std::move(f));
 }
 

src/core/lib/promise/map.h

@@ -35,7 +35,7 @@ namespace promise_detail {
 template <typename Promise, typename Fn>
 class Map {
  public:
-  Map(Promise promise, Fn fn)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Map(Promise promise, Fn fn)
       : promise_(std::move(promise)), fn_(std::move(fn)) {}
 
   Map(const Map&) = delete;
@@ -49,7 +49,7 @@ class Map {
   using Result =
       RemoveCVRef<decltype(std::declval<Fn>()(std::declval<PromiseResult>()))>;
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     Poll<PromiseResult> r = promise_();
     if (auto* p = r.value_if_ready()) {
       return fn_(std::move(*p));
@@ -68,7 +68,8 @@ class Map {
 // Takes a promise, and a synchronous function to mutate its result, and
 // returns a promise.
 template <typename Promise, typename Fn>
-promise_detail::Map<Promise, Fn> Map(Promise promise, Fn fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Map<Promise, Fn> Map(
+    Promise promise, Fn fn) {
   return promise_detail::Map<Promise, Fn>(std::move(promise), std::move(fn));
 }
 
@@ -76,7 +77,7 @@ promise_detail::Map<Promise, Fn> Map(Promise promise, Fn fn) {
 // and a bool indicating whether there was ever a Pending{} value observed from
 // polling.
 template <typename Promise>
-auto CheckDelayed(Promise promise) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto CheckDelayed(Promise promise) {
   using P = promise_detail::PromiseLike<Promise>;
   return [delayed = false, promise = P(std::move(promise))]() mutable
          -> Poll<std::tuple<typename P::Result, bool>> {
@@ -93,13 +94,13 @@ auto CheckDelayed(Promise promise) {
 template <size_t kElem>
 struct JustElem {
   template <typename... A>
-  auto operator()(std::tuple<A...>&& t) const
-      -> decltype(std::get<kElem>(std::forward<std::tuple<A...>>(t))) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()(std::tuple<A...>&& t)
+      const -> decltype(std::get<kElem>(std::forward<std::tuple<A...>>(t))) {
     return std::get<kElem>(std::forward<std::tuple<A...>>(t));
   }
   template <typename... A>
-  auto operator()(const std::tuple<A...>& t) const
-      -> decltype(std::get<kElem>(t)) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()(
+      const std::tuple<A...>& t) const -> decltype(std::get<kElem>(t)) {
     return std::get<kElem>(t);
   }
 };

src/core/lib/promise/poll.h

@@ -32,12 +32,18 @@ namespace grpc_core {
 // Allows writing 'return Pending{}' and with automatic conversions gets
 // upgraded to a Poll<> object.
 struct Pending {};
-inline bool operator==(const Pending&, const Pending&) { return true; }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(const Pending&,
+                                                            const Pending&) {
+  return true;
+}
 
 // A type that contains no value. Useful for simulating 'void' in promises that
 // always need to return some kind of value.
 struct Empty {};
-inline bool operator==(const Empty&, const Empty&) { return true; }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(const Empty&,
+                                                            const Empty&) {
+  return true;
+}
 
 // The result of polling a Promise once.
 //
@@ -47,15 +53,17 @@ template <typename T>
 class Poll {
  public:
   // NOLINTNEXTLINE(google-explicit-constructor)
-  Poll(Pending) : ready_(false) {}
-  Poll() : ready_(false) {}
-  Poll(const Poll& other) : ready_(other.ready_) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(Pending) : ready_(false) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll() : ready_(false) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(const Poll& other)
+      : ready_(other.ready_) {
     if (ready_) Construct(&value_, other.value_);
   }
-  Poll(Poll&& other) noexcept : ready_(other.ready_) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(Poll&& other) noexcept
+      : ready_(other.ready_) {
     if (ready_) Construct(&value_, std::move(other.value_));
   }
-  Poll& operator=(const Poll& other) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll& operator=(const Poll& other) {
     if (ready_) {
       if (other.ready_) {
         value_ = other.value_;
@@ -69,7 +77,7 @@ class Poll {
     }
     return *this;
   }
-  Poll& operator=(Poll&& other) noexcept {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll& operator=(Poll&& other) noexcept {
     if (ready_) {
       if (other.ready_) {
         value_ = std::move(other.value_);
@@ -85,34 +93,36 @@ class Poll {
   }
   template <typename U>
   // NOLINTNEXTLINE(google-explicit-constructor)
-  Poll(U value) : ready_(true) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(U value) : ready_(true) {
     Construct(&value_, std::move(value));
   }
   // NOLINTNEXTLINE(google-explicit-constructor)
-  Poll(T&& value) : ready_(true) { Construct(&value_, std::forward<T>(value)); }
-  ~Poll() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(T&& value) : ready_(true) {
+    Construct(&value_, std::forward<T>(value));
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Poll() {
     if (ready_) Destruct(&value_);
   }
 
-  bool pending() const { return !ready_; }
-  bool ready() const { return ready_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool pending() const { return !ready_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool ready() const { return ready_; }
 
-  T& value() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T& value() {
     DCHECK(ready());
     return value_;
   }
 
-  const T& value() const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const T& value() const {
     DCHECK(ready());
     return value_;
   }
 
-  T* value_if_ready() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T* value_if_ready() {
     if (ready()) return &value_;
     return nullptr;
   }
 
-  const T* value_if_ready() const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const T* value_if_ready() const {
     if (ready()) return &value_;
     return nullptr;
   }
@@ -143,31 +153,31 @@ template <>
 class Poll<Empty> {
  public:
   // NOLINTNEXTLINE(google-explicit-constructor)
-  Poll(Pending) : ready_(false) {}
-  Poll() : ready_(false) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(Pending) : ready_(false) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll() : ready_(false) {}
   Poll(const Poll& other) = default;
   Poll(Poll&& other) noexcept = default;
   Poll& operator=(const Poll& other) = default;
   Poll& operator=(Poll&& other) = default;
   // NOLINTNEXTLINE(google-explicit-constructor)
-  Poll(Empty) : ready_(true) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll(Empty) : ready_(true) {}
   ~Poll() = default;
 
-  bool pending() const { return !ready_; }
-  bool ready() const { return ready_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool pending() const { return !ready_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool ready() const { return ready_; }
 
-  Empty value() const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Empty value() const {
     DCHECK(ready());
     return Empty{};
   }
 
-  Empty* value_if_ready() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Empty* value_if_ready() {
     static Empty value;
     if (ready()) return &value;
     return nullptr;
   }
 
-  const Empty* value_if_ready() const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const Empty* value_if_ready() const {
     static Empty value;
     if (ready()) return &value;
     return nullptr;
@@ -195,17 +205,22 @@ class Poll<Poll<T>>;
 template <typename T>
 struct PollTraits {
   using Type = T;
-  static constexpr bool is_poll() { return false; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static constexpr bool is_poll() {
+    return false;
+  }
 };
 
 template <typename T>
 struct PollTraits<Poll<T>> {
   using Type = T;
-  static constexpr bool is_poll() { return true; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static constexpr bool is_poll() {
+    return true;
+  }
 };
 
 template <typename T>
-bool operator==(const Poll<T>& a, const Poll<T>& b) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator==(const Poll<T>& a,
+                                                     const Poll<T>& b) {
   if (a.pending() && b.pending()) return true;
   if (a.ready() && b.ready()) return a.value() == b.value();
   return false;
@@ -216,7 +231,7 @@ struct PollCastImpl;
 
 template <typename T, typename U>
 struct PollCastImpl<T, Poll<U>> {
-  static Poll<T> Cast(Poll<U>&& poll) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Cast(Poll<U>&& poll) {
     if (poll.pending()) return Pending{};
     return static_cast<T>(std::move(poll.value()));
   }
@@ -224,21 +239,27 @@ struct PollCastImpl<T, Poll<U>> {
 
 template <typename T, typename U>
 struct PollCastImpl<T, U, std::enable_if<!PollTraits<U>::is_poll()>> {
-  static Poll<T> Cast(U&& poll) { return Poll<T>(T(std::move(poll))); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Cast(U&& poll) {
+    return Poll<T>(T(std::move(poll)));
+  }
 };
 
 template <typename T>
 struct PollCastImpl<T, T> {
-  static Poll<T> Cast(T&& poll) { return Poll<T>(std::move(poll)); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Cast(T&& poll) {
+    return Poll<T>(std::move(poll));
+  }
 };
 
 template <typename T>
 struct PollCastImpl<T, Poll<T>> {
-  static Poll<T> Cast(Poll<T>&& poll) { return std::move(poll); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<T> Cast(Poll<T>&& poll) {
+    return std::move(poll);
+  }
 };
 
 template <typename T, typename U>
-Poll<T> poll_cast(U poll) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<T> poll_cast(U poll) {
   return PollCastImpl<T, U>::Cast(std::move(poll));
 }
 

src/core/lib/promise/promise.h

@@ -58,9 +58,12 @@ namespace promise_detail {
 template <typename T>
 class Immediate {
  public:
-  explicit Immediate(T value) : value_(std::move(value)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Immediate(T value)
+      : value_(std::move(value)) {}
 
-  Poll<T> operator()() { return std::move(value_); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<T> operator()() {
+    return std::move(value_);
+  }
 
  private:
   T value_;
@@ -69,13 +72,16 @@ class Immediate {
 
 // Return \a value immediately
 template <typename T>
-promise_detail::Immediate<T> Immediate(T value) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Immediate<T> Immediate(
+    T value) {
   return promise_detail::Immediate<T>(std::move(value));
 }
 
 // Return status Ok immediately
 struct ImmediateOkStatus {
-  Poll<absl::Status> operator()() { return absl::OkStatus(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<absl::Status> operator()() {
+    return absl::OkStatus();
+  }
 };
 
 // Typecheck that a promise returns the expected return type.
@@ -84,7 +90,7 @@ struct ImmediateOkStatus {
 // should fail to compile. When modifying this code these should be uncommented
 // and their miscompilation verified.
 template <typename T, typename F>
-auto WithResult(F f) ->
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto WithResult(F f) ->
     typename std::enable_if<std::is_same<decltype(f()), Poll<T>>::value,
                             F>::type {
   return f;

src/core/lib/promise/race.h

@@ -32,10 +32,11 @@ class Race<Promise, Promises...> {
  public:
   using Result = decltype(std::declval<Promise>()());
 
-  explicit Race(Promise promise, Promises... promises)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Race(Promise promise,
+                                                     Promises... promises)
       : promise_(std::move(promise)), next_(std::move(promises)...) {}
 
-  Result operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Result operator()() {
     // Check our own promise.
     auto r = promise_();
     if (r.pending()) {
@@ -57,8 +58,11 @@ template <typename Promise>
 class Race<Promise> {
  public:
   using Result = decltype(std::declval<Promise>()());
-  explicit Race(Promise promise) : promise_(std::move(promise)) {}
-  Result operator()() { return promise_(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Race(Promise promise)
+      : promise_(std::move(promise)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Result operator()() {
+    return promise_();
+  }
 
  private:
   Promise promise_;
@@ -70,7 +74,8 @@ class Race<Promise> {
 /// If two results are simultaneously available, bias towards the first result
 /// listed.
 template <typename... Promises>
-promise_detail::Race<Promises...> Race(Promises... promises) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Race<Promises...> Race(
+    Promises... promises) {
   return promise_detail::Race<Promises...>(std::move(promises)...);
 }
 

src/core/lib/promise/seq.h

@@ -36,22 +36,27 @@ struct SeqTraits {
   using UnwrappedType = T;
   using WrappedType = T;
   template <typename Next>
-  static auto CallFactory(Next* next, T&& value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(Next* next,
+                                                               T&& value) {
     return next->Make(std::forward<T>(value));
   }
-  static bool IsOk(const T&) { return true; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T&) {
+    return true;
+  }
   static const char* ErrorString(const T&) { abort(); }
   template <typename R>
   static R ReturnValue(T&&) {
     abort();
   }
   template <typename F, typename Elem>
-  static auto CallSeqFactory(F& f, Elem&& elem, T&& value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(F& f,
+                                                                  Elem&& elem,
+                                                                  T&& value) {
     return f(std::forward<Elem>(elem), std::forward<T>(value));
   }
   template <typename Result, typename PriorResult, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(PriorResult prior,
-                                            RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(PriorResult prior, RunNext run_next) {
     return run_next(std::move(prior));
   }
 };
@@ -59,11 +64,15 @@ struct SeqTraits {
 template <typename P, typename... Fs>
 class Seq {
  public:
-  explicit Seq(P&& promise, Fs&&... factories, DebugLocation whence)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Seq(P&& promise,
+                                                    Fs&&... factories,
+                                                    DebugLocation whence)
       : state_(std::forward<P>(promise), std::forward<Fs>(factories)...,
                whence) {}
 
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   SeqState<SeqTraits, P, Fs...> state_;
@@ -98,32 +107,33 @@ struct SeqIterResultTraits {
 // etc
 // Return the final value.
 template <typename F>
-F Seq(F functor) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION F Seq(F functor) {
   return functor;
 }
 
 template <typename F0, typename F1>
-promise_detail::Seq<F0, F1> Seq(F0 f0, F1 f1, DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Seq<F0, F1> Seq(
+    F0 f0, F1 f1, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1>(std::move(f0), std::move(f1), whence);
 }
 
 template <typename F0, typename F1, typename F2>
-promise_detail::Seq<F0, F1, F2> Seq(F0 f0, F1 f1, F2 f2,
-                                    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Seq<F0, F1, F2> Seq(
+    F0 f0, F1 f1, F2 f2, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2>(std::move(f0), std::move(f1),
                                          std::move(f2), whence);
 }
 
 template <typename F0, typename F1, typename F2, typename F3>
-promise_detail::Seq<F0, F1, F2, F3> Seq(F0 f0, F1 f1, F2 f2, F3 f3,
-                                        DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Seq<F0, F1, F2, F3> Seq(
+    F0 f0, F1 f1, F2 f2, F3 f3, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), whence);
 }
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4>
-promise_detail::Seq<F0, F1, F2, F3, F4> Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4,
-                                            DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Seq<F0, F1, F2, F3, F4>
+Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4>(std::move(f0), std::move(f1),
                                                  std::move(f2), std::move(f3),
                                                  std::move(f4), whence);
@@ -131,9 +141,8 @@ promise_detail::Seq<F0, F1, F2, F3, F4> Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4,
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5> Seq(F0 f0, F1 f1, F2 f2, F3 f3,
-                                                F4 f4, F5 f5,
-                                                DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Seq<F0, F1, F2, F3, F4, F5>
+Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), whence);
@@ -141,9 +150,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5> Seq(F0 f0, F1 f1, F2 f2, F3 f3,
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6> Seq(F0 f0, F1 f1, F2 f2, F3 f3,
-                                                    F4 f4, F5 f5, F6 f6,
-                                                    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6,
+        DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), whence);
@@ -151,9 +161,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6> Seq(F0 f0, F1 f1, F2 f2, F3 f3,
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7> Seq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7,
-    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7,
+        DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), whence);
@@ -161,9 +172,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7> Seq(
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7, typename F8>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8> Seq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8,
-    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8,
+        DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), std::move(f8), whence);
@@ -171,9 +183,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8> Seq(
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7, typename F8, typename F9>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> Seq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
-    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
+        DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), std::move(f8), std::move(f9),
@@ -183,9 +196,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9> Seq(
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7, typename F8, typename F9,
           typename F10>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> Seq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
-    F10 f10, DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
+        F10 f10, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), std::move(f8), std::move(f9),
@@ -195,9 +209,10 @@ promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10> Seq(
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7, typename F8, typename F9,
           typename F10, typename F11>
-promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11> Seq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
-    F10 f10, F11 f11, DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11>
+    Seq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7, F8 f8, F9 f9,
+        F10 f10, F11 f11, DebugLocation whence = {}) {
   return promise_detail::Seq<F0, F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), std::move(f8), std::move(f9),

src/core/lib/promise/status_flag.h

@@ -42,43 +42,58 @@ struct Success {
   }
 };
 
-inline bool IsStatusOk(Failure) { return false; }
-inline bool IsStatusOk(Success) { return true; }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(Failure) {
+  return false;
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(Success) {
+  return true;
+}
 
 template <>
 struct StatusCastImpl<absl::Status, Success> {
-  static absl::Status Cast(Success) { return absl::OkStatus(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(Success) {
+    return absl::OkStatus();
+  }
 };
 
 template <>
 struct StatusCastImpl<absl::Status, const Success&> {
-  static absl::Status Cast(Success) { return absl::OkStatus(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(Success) {
+    return absl::OkStatus();
+  }
 };
 
 template <>
 struct StatusCastImpl<absl::Status, Failure> {
-  static absl::Status Cast(Failure) { return absl::CancelledError(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(Failure) {
+    return absl::CancelledError();
+  }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::StatusOr<T>, Failure> {
-  static absl::StatusOr<T> Cast(Failure) { return absl::CancelledError(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(Failure) {
+    return absl::CancelledError();
+  }
 };
 
 // A boolean representing whether an operation succeeded (true) or failed
 // (false).
 class StatusFlag {
  public:
-  StatusFlag() : value_(true) {}
-  explicit StatusFlag(bool value) : value_(value) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION StatusFlag() : value_(true) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit StatusFlag(bool value)
+      : value_(value) {}
   // NOLINTNEXTLINE(google-explicit-constructor)
-  StatusFlag(Failure) : value_(false) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION StatusFlag(Failure) : value_(false) {}
   // NOLINTNEXTLINE(google-explicit-constructor)
-  StatusFlag(Success) : value_(true) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION StatusFlag(Success) : value_(true) {}
 
-  bool ok() const { return value_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool ok() const { return value_; }
 
-  bool operator==(StatusFlag other) const { return value_ == other.value_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator==(StatusFlag other) const {
+    return value_ == other.value_;
+  }
   std::string ToString() const { return value_ ? "ok" : "failed"; }
 
   template <typename Sink>
@@ -94,42 +109,73 @@ class StatusFlag {
   bool value_;
 };
 
-inline bool operator==(StatusFlag flag, Failure) { return !flag.ok(); }
-inline bool operator==(Failure, StatusFlag flag) { return !flag.ok(); }
-inline bool operator==(StatusFlag flag, Success) { return flag.ok(); }
-inline bool operator==(Success, StatusFlag flag) { return flag.ok(); }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(StatusFlag flag,
+                                                            Failure) {
+  return !flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(Failure,
+                                                            StatusFlag flag) {
+  return !flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(StatusFlag flag,
+                                                            Success) {
+  return flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator==(Success,
+                                                            StatusFlag flag) {
+  return flag.ok();
+}
 
-inline bool operator!=(StatusFlag flag, Failure) { return flag.ok(); }
-inline bool operator!=(Failure, StatusFlag flag) { return flag.ok(); }
-inline bool operator!=(StatusFlag flag, Success) { return !flag.ok(); }
-inline bool operator!=(Success, StatusFlag flag) { return !flag.ok(); }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator!=(StatusFlag flag,
+                                                            Failure) {
+  return flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator!=(Failure,
+                                                            StatusFlag flag) {
+  return flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator!=(StatusFlag flag,
+                                                            Success) {
+  return !flag.ok();
+}
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool operator!=(Success,
+                                                            StatusFlag flag) {
+  return !flag.ok();
+}
 
-inline bool IsStatusOk(const StatusFlag& flag) { return flag.ok(); }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const StatusFlag& flag) {
+  return flag.ok();
+}
 
 template <>
 struct StatusCastImpl<absl::Status, StatusFlag> {
-  static absl::Status Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      StatusFlag flag) {
     return flag.ok() ? absl::OkStatus() : absl::CancelledError();
   }
 };
 
 template <>
 struct StatusCastImpl<absl::Status, StatusFlag&> {
-  static absl::Status Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      StatusFlag flag) {
     return flag.ok() ? absl::OkStatus() : absl::CancelledError();
   }
 };
 
 template <>
 struct StatusCastImpl<absl::Status, const StatusFlag&> {
-  static absl::Status Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      StatusFlag flag) {
     return flag.ok() ? absl::OkStatus() : absl::CancelledError();
   }
 };
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, StatusFlag> {
-  static absl::StatusOr<T> Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      StatusFlag flag) {
     DCHECK(!flag.ok());
     return absl::CancelledError();
   }
@@ -137,7 +183,8 @@ struct FailureStatusCastImpl<absl::StatusOr<T>, StatusFlag> {
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, StatusFlag&> {
-  static absl::StatusOr<T> Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      StatusFlag flag) {
     DCHECK(!flag.ok());
     return absl::CancelledError();
   }
@@ -145,7 +192,8 @@ struct FailureStatusCastImpl<absl::StatusOr<T>, StatusFlag&> {
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, const StatusFlag&> {
-  static absl::StatusOr<T> Cast(StatusFlag flag) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      StatusFlag flag) {
     DCHECK(!flag.ok());
     return absl::CancelledError();
   }
@@ -162,31 +210,48 @@ class ValueOrFailure {
   // NOLINTNEXTLINE(google-explicit-constructor)
   ValueOrFailure(StatusFlag status) { CHECK(!status.ok()); }
 
-  static ValueOrFailure FromOptional(absl::optional<T> value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static ValueOrFailure FromOptional(
+      absl::optional<T> value) {
     return ValueOrFailure{std::move(value)};
   }
 
-  bool ok() const { return value_.has_value(); }
-  StatusFlag status() const { return StatusFlag(ok()); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool ok() const {
+    return value_.has_value();
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION StatusFlag status() const {
+    return StatusFlag(ok());
+  }
 
-  const T& value() const { return value_.value(); }
-  T& value() { return value_.value(); }
-  const T& operator*() const { return *value_; }
-  T& operator*() { return *value_; }
-  const T* operator->() const { return &*value_; }
-  T* operator->() { return &*value_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const T& value() const {
+    return value_.value();
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T& value() { return value_.value(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const T& operator*() const {
+    return *value_;
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T& operator*() { return *value_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION const T* operator->() const {
+    return &*value_;
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T* operator->() { return &*value_; }
 
-  bool operator==(const ValueOrFailure& other) const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator==(
+      const ValueOrFailure& other) const {
     return value_ == other.value_;
   }
 
-  bool operator!=(const ValueOrFailure& other) const {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator!=(
+      const ValueOrFailure& other) const {
     return value_ != other.value_;
   }
 
-  bool operator==(const T& other) const { return value_ == other; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator==(const T& other) const {
+    return value_ == other;
+  }
 
-  bool operator!=(const T& other) const { return value_ != other; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool operator!=(const T& other) const {
+    return value_ != other;
+  }
 
  private:
   absl::optional<T> value_;
@@ -203,18 +268,21 @@ inline std::ostream& operator<<(std::ostream& os,
 }
 
 template <typename T>
-inline bool IsStatusOk(const ValueOrFailure<T>& value) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const ValueOrFailure<T>& value) {
   return value.ok();
 }
 
 template <typename T>
-inline T TakeValue(ValueOrFailure<T>&& value) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline T TakeValue(
+    ValueOrFailure<T>&& value) {
   return std::move(value.value());
 }
 
 template <typename T>
 struct StatusCastImpl<absl::StatusOr<T>, ValueOrFailure<T>> {
-  static absl::StatusOr<T> Cast(ValueOrFailure<T> value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      ValueOrFailure<T> value) {
     return value.ok() ? absl::StatusOr<T>(std::move(value.value()))
                       : absl::CancelledError();
   }
@@ -222,14 +290,15 @@ struct StatusCastImpl<absl::StatusOr<T>, ValueOrFailure<T>> {
 
 template <typename T>
 struct StatusCastImpl<ValueOrFailure<T>, Failure> {
-  static ValueOrFailure<T> Cast(Failure) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static ValueOrFailure<T> Cast(Failure) {
     return ValueOrFailure<T>(Failure{});
   }
 };
 
 template <typename T>
 struct StatusCastImpl<ValueOrFailure<T>, StatusFlag&> {
-  static ValueOrFailure<T> Cast(StatusFlag f) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static ValueOrFailure<T> Cast(
+      StatusFlag f) {
     CHECK(!f.ok());
     return ValueOrFailure<T>(Failure{});
   }
@@ -237,7 +306,8 @@ struct StatusCastImpl<ValueOrFailure<T>, StatusFlag&> {
 
 template <typename T>
 struct StatusCastImpl<ValueOrFailure<T>, StatusFlag> {
-  static ValueOrFailure<T> Cast(StatusFlag f) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static ValueOrFailure<T> Cast(
+      StatusFlag f) {
     CHECK(!f.ok());
     return ValueOrFailure<T>(Failure{});
   }

src/core/lib/promise/try_join.h

@@ -51,44 +51,56 @@ struct TryJoinTraits {
   template <typename T>
   using ResultType = Result<absl::remove_reference_t<T>>;
   template <typename T>
-  static bool IsOk(const absl::StatusOr<T>& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(
+      const absl::StatusOr<T>& x) {
+    return x.ok();
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const absl::Status& x) {
+    return x.ok();
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(StatusFlag x) {
     return x.ok();
   }
-  static bool IsOk(const absl::Status& x) { return x.ok(); }
-  static bool IsOk(StatusFlag x) { return x.ok(); }
   template <typename T>
-  static bool IsOk(const ValueOrFailure<T>& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(
+      const ValueOrFailure<T>& x) {
     return x.ok();
   }
   template <typename T>
-  static T Unwrapped(absl::StatusOr<T> x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T Unwrapped(absl::StatusOr<T> x) {
     return std::move(*x);
   }
   template <typename T>
-  static T Unwrapped(ValueOrFailure<T> x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T Unwrapped(ValueOrFailure<T> x) {
     return std::move(*x);
   }
-  static Empty Unwrapped(absl::Status) { return Empty{}; }
-  static Empty Unwrapped(StatusFlag) { return Empty{}; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(absl::Status) {
+    return Empty{};
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(StatusFlag) {
+    return Empty{};
+  }
   template <typename R, typename T>
-  static R EarlyReturn(absl::StatusOr<T> x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(
+      absl::StatusOr<T> x) {
     return x.status();
   }
   template <typename R>
-  static R EarlyReturn(absl::Status x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(absl::Status x) {
     return FailureStatusCast<R>(std::move(x));
   }
   template <typename R>
-  static R EarlyReturn(StatusFlag x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(StatusFlag x) {
     return FailureStatusCast<R>(x);
   }
   template <typename R, typename T>
-  static R EarlyReturn(const ValueOrFailure<T>& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(
+      const ValueOrFailure<T>& x) {
     CHECK(!x.ok());
     return FailureStatusCast<R>(Failure{});
   }
   template <typename... A>
-  static auto FinalReturn(A&&... a) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto FinalReturn(A&&... a) {
     return Result<std::tuple<A...>>(std::make_tuple(std::forward<A>(a)...));
   }
 };
@@ -97,8 +109,11 @@ struct TryJoinTraits {
 template <template <typename> class R, typename... Promises>
 class TryJoin {
  public:
-  explicit TryJoin(Promises... promises) : state_(std::move(promises)...) {}
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit TryJoin(Promises... promises)
+      : state_(std::move(promises)...) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   JoinState<TryJoinTraits<R>, Promises...> state_;
@@ -107,7 +122,7 @@ class TryJoin {
 template <template <typename> class R>
 struct WrapInStatusOrTuple {
   template <typename T>
-  R<std::tuple<T>> operator()(R<T> x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION R<std::tuple<T>> operator()(R<T> x) {
     if (!x.ok()) return x.status();
     return std::make_tuple(std::move(*x));
   }
@@ -119,12 +134,13 @@ struct WrapInStatusOrTuple {
 // If any fail, cancel the rest and return the failure.
 // If all succeed, return Ok(tuple-of-results).
 template <template <typename> class R, typename... Promises>
-promise_detail::TryJoin<R, Promises...> TryJoin(Promises... promises) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::TryJoin<R, Promises...>
+TryJoin(Promises... promises) {
   return promise_detail::TryJoin<R, Promises...>(std::move(promises)...);
 }
 
 template <template <typename> class R, typename F>
-auto TryJoin(F promise) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto TryJoin(F promise) {
   return Map(promise, promise_detail::WrapInStatusOrTuple<R>{});
 }
 

src/core/lib/promise/try_seq.h

@@ -43,22 +43,28 @@ struct TrySeqTraitsWithSfinae {
   using UnwrappedType = T;
   using WrappedType = absl::StatusOr<T>;
   template <typename Next>
-  static auto CallFactory(Next* next, T&& value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(Next* next,
+                                                               T&& value) {
     return next->Make(std::forward<T>(value));
   }
-  static bool IsOk(const T&) { return true; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T&) {
+    return true;
+  }
   static const char* ErrorString(const T&) { abort(); }
   template <typename R>
   static R ReturnValue(T&&) {
     abort();
   }
   template <typename F, typename Elem>
-  static auto CallSeqFactory(F& f, Elem&& elem, T&& value)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(F& f,
+                                                                  Elem&& elem,
+                                                                  T&& value)
       -> decltype(f(std::forward<Elem>(elem), std::forward<T>(value))) {
     return f(std::forward<Elem>(elem), std::forward<T>(value));
   }
   template <typename Result, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(T prior, RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(T prior, RunNext run_next) {
     return run_next(std::move(prior));
   }
 };
@@ -68,25 +74,31 @@ struct TrySeqTraitsWithSfinae<absl::StatusOr<T>> {
   using UnwrappedType = T;
   using WrappedType = absl::StatusOr<T>;
   template <typename Next>
-  static auto CallFactory(Next* next, absl::StatusOr<T>&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(
+      Next* next, absl::StatusOr<T>&& status) {
     return next->Make(std::move(*status));
   }
-  static bool IsOk(const absl::StatusOr<T>& status) { return status.ok(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(
+      const absl::StatusOr<T>& status) {
+    return status.ok();
+  }
   static std::string ErrorString(const absl::StatusOr<T>& status) {
     return status.status().ToString();
   }
   template <typename R>
-  static R ReturnValue(absl::StatusOr<T>&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R ReturnValue(
+      absl::StatusOr<T>&& status) {
     return FailureStatusCast<R>(status.status());
   }
   template <typename F, typename Elem>
-  static auto CallSeqFactory(F& f, Elem&& elem, absl::StatusOr<T> value)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(
+      F& f, Elem&& elem, absl::StatusOr<T> value)
       -> decltype(f(std::forward<Elem>(elem), std::move(*value))) {
     return f(std::forward<Elem>(elem), std::move(*value));
   }
   template <typename Result, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(absl::StatusOr<T> prior,
-                                            RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(absl::StatusOr<T> prior, RunNext run_next) {
     if (!prior.ok()) return FailureStatusCast<Result>(prior.status());
     return run_next(std::move(prior));
   }
@@ -128,19 +140,24 @@ struct TrySeqTraitsWithSfinae<
   using UnwrappedType = void;
   using WrappedType = T;
   template <typename Next>
-  static auto CallFactory(Next* next, T&&) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(Next* next,
+                                                               T&&) {
     return next->Make();
   }
-  static bool IsOk(const T& status) { return IsStatusOk(status); }
-  static std::string ErrorString(const T& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T& status) {
+    return IsStatusOk(status);
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static std::string ErrorString(
+      const T& status) {
     return IsStatusOk(status) ? "OK" : "FAILED";
   }
   template <typename R>
-  static R ReturnValue(T&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R ReturnValue(T&& status) {
     return FailureStatusCast<R>(std::move(status));
   }
   template <typename Result, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(T prior, RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(T prior, RunNext run_next) {
     if (!IsStatusOk(prior)) return Result(std::move(prior));
     return run_next(std::move(prior));
   }
@@ -154,20 +171,25 @@ struct TrySeqTraitsWithSfinae<
   using UnwrappedType = decltype(TakeValue(std::declval<T>()));
   using WrappedType = T;
   template <typename Next>
-  static auto CallFactory(Next* next, T&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(Next* next,
+                                                               T&& status) {
     return next->Make(TakeValue(std::forward<T>(status)));
   }
-  static bool IsOk(const T& status) { return IsStatusOk(status); }
-  static std::string ErrorString(const T& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T& status) {
+    return IsStatusOk(status);
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static std::string ErrorString(
+      const T& status) {
     return IsStatusOk(status) ? "OK" : "FAILED";
   }
   template <typename R>
-  static R ReturnValue(T&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R ReturnValue(T&& status) {
     DCHECK(!IsStatusOk(status));
     return FailureStatusCast<R>(status.status());
   }
   template <typename Result, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(T prior, RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(T prior, RunNext run_next) {
     if (!IsStatusOk(prior)) return Result(std::move(prior));
     return run_next(std::move(prior));
   }
@@ -177,20 +199,25 @@ struct TrySeqTraitsWithSfinae<absl::Status> {
   using UnwrappedType = void;
   using WrappedType = absl::Status;
   template <typename Next>
-  static auto CallFactory(Next* next, absl::Status&&) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(Next* next,
+                                                               absl::Status&&) {
     return next->Make();
   }
-  static bool IsOk(const absl::Status& status) { return status.ok(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(
+      const absl::Status& status) {
+    return status.ok();
+  }
   static std::string ErrorString(const absl::Status& status) {
     return status.ToString();
   }
   template <typename R>
-  static R ReturnValue(absl::Status&& status) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R ReturnValue(
+      absl::Status&& status) {
     return FailureStatusCast<R>(std::move(status));
   }
   template <typename Result, typename RunNext>
-  static Poll<Result> CheckResultAndRunNext(absl::Status prior,
-                                            RunNext run_next) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(absl::Status prior, RunNext run_next) {
     if (!prior.ok()) return StatusCast<Result>(std::move(prior));
     return run_next(std::move(prior));
   }
@@ -202,11 +229,15 @@ using TrySeqTraits = TrySeqTraitsWithSfinae<T>;
 template <typename P, typename... Fs>
 class TrySeq {
  public:
-  explicit TrySeq(P&& promise, Fs&&... factories, DebugLocation whence)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit TrySeq(P&& promise,
+                                                       Fs&&... factories,
+                                                       DebugLocation whence)
       : state_(std::forward<P>(promise), std::forward<Fs>(factories)...,
                whence) {}
 
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   SeqState<TrySeqTraits, P, Fs...> state_;
@@ -247,33 +278,33 @@ struct TrySeqIterResultTraits {
 // Status to indicate only success/failure. In the case of returning Status,
 // the construction functors take no arguments.
 template <typename F>
-F TrySeq(F functor) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION F TrySeq(F functor) {
   return functor;
 }
 
 template <typename F0, typename F1>
-promise_detail::TrySeq<F0, F1> TrySeq(F0 f0, F1 f1, DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::TrySeq<F0, F1> TrySeq(
+    F0 f0, F1 f1, DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1>(std::move(f0), std::move(f1), whence);
 }
 
 template <typename F0, typename F1, typename F2>
-promise_detail::TrySeq<F0, F1, F2> TrySeq(F0 f0, F1 f1, F2 f2,
-                                          DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::TrySeq<F0, F1, F2> TrySeq(
+    F0 f0, F1 f1, F2 f2, DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2>(std::move(f0), std::move(f1),
                                             std::move(f2), whence);
 }
 
 template <typename F0, typename F1, typename F2, typename F3>
-promise_detail::TrySeq<F0, F1, F2, F3> TrySeq(F0 f0, F1 f1, F2 f2, F3 f3,
-                                              DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::TrySeq<F0, F1, F2, F3>
+TrySeq(F0 f0, F1 f1, F2 f2, F3 f3, DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2, F3>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), whence);
 }
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4>
-promise_detail::TrySeq<F0, F1, F2, F3, F4> TrySeq(F0 f0, F1 f1, F2 f2, F3 f3,
-                                                  F4 f4,
-                                                  DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::TrySeq<F0, F1, F2, F3, F4>
+TrySeq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2, F3, F4>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       whence);
@@ -281,8 +312,10 @@ promise_detail::TrySeq<F0, F1, F2, F3, F4> TrySeq(F0 f0, F1 f1, F2 f2, F3 f3,
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5>
-promise_detail::TrySeq<F0, F1, F2, F3, F4, F5> TrySeq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::TrySeq<F0, F1, F2, F3, F4, F5>
+    TrySeq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5,
+           DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2, F3, F4, F5>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), whence);
@@ -290,9 +323,10 @@ promise_detail::TrySeq<F0, F1, F2, F3, F4, F5> TrySeq(
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6>
-promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6> TrySeq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6,
-    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6>
+    TrySeq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6,
+           DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), whence);
@@ -300,9 +334,10 @@ promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6> TrySeq(
 
 template <typename F0, typename F1, typename F2, typename F3, typename F4,
           typename F5, typename F6, typename F7>
-promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6, F7> TrySeq(
-    F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7,
-    DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6, F7>
+    TrySeq(F0 f0, F1 f1, F2 f2, F3 f3, F4 f4, F5 f5, F6 f6, F7 f7,
+           DebugLocation whence = {}) {
   return promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6, F7>(
       std::move(f0), std::move(f1), std::move(f2), std::move(f3), std::move(f4),
       std::move(f5), std::move(f6), std::move(f7), whence);
@@ -317,8 +352,10 @@ promise_detail::TrySeq<F0, F1, F2, F3, F4, F5, F6, F7> TrySeq(
 //   }
 //   return argument;
 template <typename Iter, typename Factory, typename Argument>
-typename promise_detail::TrySeqIterResultTraits<Iter, Factory, Argument>::Result
-TrySeqIter(Iter begin, Iter end, Argument argument, Factory factory) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+    typename promise_detail::TrySeqIterResultTraits<Iter, Factory,
+                                                    Argument>::Result
+    TrySeqIter(Iter begin, Iter end, Argument argument, Factory factory) {
   using Result =
       typename promise_detail::TrySeqIterResultTraits<Iter, Factory,
                                                       Argument>::Result;

tools/codegen/core/gen_join.py

@@ -33,20 +33,20 @@ struct JoinState<Traits, ${",".join(f"P{i}" for i in range(0,n))}> {
   };
 % endfor
   GPR_NO_UNIQUE_ADDRESS BitSet<${n}> ready;
-  JoinState(${",".join(f"P{i}&& p{i}" for i in range(0,n))}) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION JoinState(${",".join(f"P{i}&& p{i}" for i in range(0,n))}) {
 % for i in range(0,n):
     Construct(&promise${i}, std::forward<P${i}>(p${i}));
 % endfor
   }
-  JoinState(const JoinState& other) {
-    CHECK(other.ready.none());
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION JoinState(const JoinState& other) {
+    DCHECK(other.ready.none());
 % for i in range(0,n):
     Construct(&promise${i}, other.promise${i});
 % endfor
   }
   JoinState& operator=(const JoinState& other) = delete;
   JoinState& operator=(JoinState&& other) = delete;
-  JoinState(JoinState&& other) noexcept : ready(other.ready) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION JoinState(JoinState&& other) noexcept : ready(other.ready) {
 % for i in range(0,n):
     if (ready.is_set(${i})) {
       Construct(&result${i}, std::move(other.result${i}));
@@ -55,7 +55,7 @@ struct JoinState<Traits, ${",".join(f"P{i}" for i in range(0,n))}> {
     }
 % endfor
   }
-  ~JoinState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~JoinState() {
 % for i in range(0,n):
     if (ready.is_set(${i})) {
       Destruct(&result${i});
@@ -66,16 +66,13 @@ struct JoinState<Traits, ${",".join(f"P{i}" for i in range(0,n))}> {
   }
   using Result = typename Traits::template ResultType<std::tuple<
       ${",".join(f"Result{i}" for i in range(0,n))}>>;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
 % for i in range(0,n):
     if (!ready.is_set(${i})) {
       GRPC_TRACE_VLOG(promise_primitives, 2) << "join[" << this << "]: begin poll joint ${i+1}/${n}";
       auto poll = promise${i}();
-      if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        auto* p = poll.value_if_ready();
-        VLOG(2) << "join[" << this << "]: joint ${i+1}/${n} "
-                << (p != nullptr ? (Traits::IsOk(*p)? "ready" : "early-error") : "pending");
-      }
+      GRPC_TRACE_VLOG(promise_primitives, 2) << "join[" << this << "]: end poll joint ${i+1}/${n} "
+                                             << (poll.pending()? "pending" : (Traits::IsOk(poll.value())? "ready" : "early-error"));
       if (auto* p = poll.value_if_ready()) {
         if (Traits::IsOk(*p)) {
           ready.set(${i});
@@ -86,7 +83,7 @@ struct JoinState<Traits, ${",".join(f"P{i}" for i in range(0,n))}> {
         }
       }
     } else if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      VLOG(2) << "join[" << this << "]: joint ${i+1}/${n} already ready";
+      GRPC_TRACE_VLOG(promise_primitives, 2) << "join[" << this << "]: joint ${i+1}/${n} already ready";
     }
 % endfor
     if (ready.all()) {
@@ -102,7 +99,7 @@ front_matter = """
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_DETAIL_JOIN_STATE_H
 #define GRPC_SRC_CORE_LIB_PROMISE_DETAIL_JOIN_STATE_H
 
-// This file is generated by tools/codegen/core/gen_seq.py
+// This file is generated by tools/codegen/core/gen_join.py
 
 #include <grpc/support/port_platform.h>
 

tools/codegen/core/gen_seq.py

@@ -85,7 +85,7 @@ union {
   GPR_NO_UNIQUE_ADDRESS State state = State::kState0;
   GPR_NO_UNIQUE_ADDRESS DebugLocation whence;
 
-  SeqState(P&& p,
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(P&& p,
            ${",".join(f"F{i}&& f{i}" for i in range(0,n-1))},
            DebugLocation whence) noexcept: whence(whence)  {
     Construct(&${"prior."*(n-1)}current_promise, std::forward<P>(p));
@@ -93,7 +93,7 @@ union {
     Construct(&${"prior."*(n-1-i)}next_factory, std::forward<F${i}>(f${i}));
 % endfor
   }
-  ~SeqState() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~SeqState() {
     switch (state) {
 % for i in range(0,n-1):
      case State::kState${i}:
@@ -109,7 +109,7 @@ tail${i}:
     Destruct(&${"prior."*(n-1-i)}next_factory);
 % endfor
   }
-  SeqState(const SeqState& other) noexcept : state(other.state), whence(other.whence) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(const SeqState& other) noexcept : state(other.state), whence(other.whence) {
     CHECK(state == State::kState0);
     Construct(&${"prior."*(n-1-i)}current_promise,
             other.${"prior."*(n-1-i)}current_promise);
@@ -119,7 +119,7 @@ tail${i}:
 % endfor
   }
   SeqState& operator=(const SeqState& other) = delete;
-  SeqState(SeqState&& other) noexcept : state(other.state), whence(other.whence) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState(SeqState&& other) noexcept : state(other.state), whence(other.whence) {
     switch (state) {
 % for i in range(0,n-1):
      case State::kState${i}:
@@ -137,8 +137,8 @@ tail${i}:
               std::move(other.${"prior."*(n-1-i)}next_factory));
 % endfor
   }
-  SeqState& operator=(SeqState&& other) = delete;
-  Poll<Result> PollOnce() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION SeqState& operator=(SeqState&& other) = delete;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollOnce() {
     switch (state) {
 % for i in range(0,n-1):
       case State::kState${i}: {
@@ -194,7 +194,7 @@ front_matter = """
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_DETAIL_SEQ_STATE_H
 #define GRPC_SRC_CORE_LIB_PROMISE_DETAIL_SEQ_STATE_H
 
-// This file is generated by tools/codegen/core/gen_join.py
+// This file is generated by tools/codegen/core/gen_seq.py
 
 #include <grpc/support/port_platform.h>