diff --git a/Makefile b/Makefile
index 2822623a221..a2789e40431 100644
--- a/Makefile
+++ b/Makefile
@@ -404,6 +404,28 @@ LIBS = m pthread ws2_32
 LDFLAGS += -pthread
 endif
 
+# If we are installing into a non-default prefix, both
+# the libraries we build, and the apps users build,
+# need to know how to find the libraries they depend on.
+# There is much gnashing of teeth about this subject.
+# It's tricky to do that without editing images during install,
+# as you don't want tests during build to find previously installed and
+# now stale libraries, etc.
+ifeq ($(SYSTEM),Linux)
+ifneq ($(prefix),/usr)
+# Linux best practice for rpath on installed files is probably:
+# 1) .pc file provides -Wl,-rpath,$(prefix)/lib
+# 2) binaries we install into $(prefix)/bin use -Wl,-rpath,$ORIGIN/../lib
+# 3) libraries we install into $(prefix)/lib use -Wl,-rpath,$ORIGIN
+# cf. https://www.akkadia.org/drepper/dsohowto.pdf
+# Doing all of that right is hard, but using -Wl,-rpath,$ORIGIN is always
+# safe, and solves problems seen in the wild.  Note that $ORIGIN
+# is a literal string interpreted much later by ld.so.  Escape it
+# here with a dollar sign so Make doesn't expand $O.
+LDFLAGS += '-Wl,-rpath,$$ORIGIN'
+endif
+endif
+
 #
 # The steps for cross-compiling are as follows:
 # First, clone and make install of grpc using the native compilers for the host.
diff --git a/templates/Makefile.template b/templates/Makefile.template
index 31cf14a71c1..71391f8139a 100644
--- a/templates/Makefile.template
+++ b/templates/Makefile.template
@@ -274,6 +274,28 @@
   LDFLAGS += -pthread
   endif
 
+  # If we are installing into a non-default prefix, both
+  # the libraries we build, and the apps users build,
+  # need to know how to find the libraries they depend on.
+  # There is much gnashing of teeth about this subject.
+  # It's tricky to do that without editing images during install,
+  # as you don't want tests during build to find previously installed and
+  # now stale libraries, etc.
+  ifeq ($(SYSTEM),Linux)
+  ifneq ($(prefix),/usr)
+  # Linux best practice for rpath on installed files is probably:
+  # 1) .pc file provides -Wl,-rpath,$(prefix)/lib
+  # 2) binaries we install into $(prefix)/bin use -Wl,-rpath,$ORIGIN/../lib
+  # 3) libraries we install into $(prefix)/lib use -Wl,-rpath,$ORIGIN
+  # cf. https://www.akkadia.org/drepper/dsohowto.pdf
+  # Doing all of that right is hard, but using -Wl,-rpath,$ORIGIN is always
+  # safe, and solves problems seen in the wild.  Note that $ORIGIN
+  # is a literal string interpreted much later by ld.so.  Escape it
+  # here with a dollar sign so Make doesn't expand $O.
+  LDFLAGS += '-Wl,-rpath,$$ORIGIN'
+  endif
+  endif
+
   #
   # The steps for cross-compiling are as follows:
   # First, clone and make install of grpc using the native compilers for the host.