diff --git a/Makefile b/Makefile
index efa08cdfb0a..4206d7b88d8 100644
--- a/Makefile
+++ b/Makefile
@@ -404,6 +404,28 @@ LIBS = m pthread ws2_32
 LDFLAGS += -pthread
 endif
 
+# If we are installing into a non-default prefix, both
+# the libraries we build, and the apps users build,
+# need to know how to find the libraries they depend on.
+# There is much gnashing of teeth about this subject.
+# It's tricky to do that without editing images during install,
+# as you don't want tests during build to find previously installed and
+# now stale libraries, etc.
+ifeq ($(SYSTEM),Linux)
+ifneq ($(prefix),/usr)
+# Linux best practice for rpath on installed files is probably:
+# 1) .pc file provides -Wl,-rpath,$(prefix)/lib
+# 2) binaries we install into $(prefix)/bin use -Wl,-rpath,$ORIGIN/../lib
+# 3) libraries we install into $(prefix)/lib use -Wl,-rpath,$ORIGIN
+# cf. https://www.akkadia.org/drepper/dsohowto.pdf
+# Doing all of that right is hard, but using -Wl,-rpath,$ORIGIN is always
+# safe, and solves problems seen in the wild.  Note that $ORIGIN
+# is a literal string interpreted much later by ld.so.  Escape it
+# here with a dollar sign so Make doesn't expand $O.
+LDFLAGS += '-Wl,-rpath,$$ORIGIN'
+endif
+endif
+
 #
 # The steps for cross-compiling are as follows:
 # First, clone and make install of grpc using the native compilers for the host.