Makefile: fix shared library resolution on linux when installed to e.g. /opt

For issue 18131.

templates/Makefile.template

@@ -274,6 +274,28 @@
   LDFLAGS += -pthread
   endif
 
+  # If we are installing into a non-default prefix, both
+  # the libraries we build, and the apps users build,
+  # need to know how to find the libraries they depend on.
+  # There is much gnashing of teeth about this subject.
+  # It's tricky to do that without editing images during install,
+  # as you don't want tests during build to find previously installed and
+  # now stale libraries, etc.
+  ifeq ($(SYSTEM),Linux)
+  ifneq ($(prefix),/usr)
+  # Linux best practice for rpath on installed files is probably:
+  # 1) .pc file provides -Wl,-rpath,$(prefix)/lib
+  # 2) binaries we install into $(prefix)/bin use -Wl,-rpath,$ORIGIN/../lib
+  # 3) libraries we install into $(prefix)/lib use -Wl,-rpath,$ORIGIN
+  # cf. https://www.akkadia.org/drepper/dsohowto.pdf
+  # Doing all of that right is hard, but using -Wl,-rpath,$ORIGIN is always
+  # safe, and solves problems seen in the wild.  Note that $ORIGIN
+  # is a literal string interpreted much later by ld.so.  Escape it
+  # here with a dollar sign so Make doesn't expand $O.
+  LDFLAGS += '-Wl,-rpath,$$ORIGIN'
+  endif
+  endif
+
   #
   # The steps for cross-compiling are as follows:
   # First, clone and make install of grpc using the native compilers for the host.