From: Stefano Rivera <stefanor@debian.org>
Date: Sat, 7 Oct 2017 09:38:58 +0200
Subject: Debian: Add a distutils option --install-layout=deb

This option:
 - installs into $prefix/dist-packages instead of $prefix/site-packages.
 - doesn't encode the python version into the egg name.

Based on cpython Debian packaging

Author: Matthias Klose <doko@debian.org>
Author: Stefano Rivera <stefanor@debian.org>
Last-Update: 2017-05-21
---
 lib-python/3/_distutils_system_mod.py              | 136 +++++++++++++++++++++
 lib-python/3/distutils/command/install.py          |  43 ++++++-
 lib-python/3/distutils/command/install_egg_info.py |  30 ++++-
 lib-python/3/distutils/sysconfig_pypy.py           |   7 +-
 lib-python/3/pydoc.py                              |   1 +
 lib-python/3/site.py                               |  22 +++-
 lib-python/3/sysconfig.py                          |  40 +++++-
 lib-python/3/test/test_sysconfig.py                |   4 +-
 8 files changed, 265 insertions(+), 18 deletions(-)
 create mode 100644 lib-python/3/_distutils_system_mod.py

diff --git a/lib-python/3/_distutils_system_mod.py b/lib-python/3/_distutils_system_mod.py
new file mode 100644
index 0000000..a2c76f8
--- /dev/null
+++ b/lib-python/3/_distutils_system_mod.py
@@ -0,0 +1,136 @@
+"""
+Apply Debian-specific patches to distutils commands.
+
+Extracts the customized behavior from patches as reported
+in pypa/distutils#2 and applies those customizations (except
+for scheme definitions) to those commands.
+
+Place this module somewhere in sys.path to take effect.
+"""
+
+import os
+import sys
+
+import distutils.sysconfig
+import distutils.command.install as orig_install
+import distutils.command.install_egg_info as orig_install_egg_info
+from distutils.command.install_egg_info import (
+    to_filename,
+    safe_name,
+    safe_version,
+    )
+from distutils.errors import DistutilsOptionError
+
+
+class install(orig_install.install):
+    user_options = list(orig_install.install.user_options) + [
+        ('install-layout=', None,
+         "installation layout to choose (known values: deb, unix)"),
+    ]
+
+    def initialize_options(self):
+        super().initialize_options()
+        self.prefix_option = None
+        self.install_layout = None
+
+    def finalize_unix(self):
+        self.prefix_option = self.prefix
+        super().finalize_unix()
+        if self.install_layout:
+            if self.install_layout.lower() in ['deb']:
+                self.select_scheme("deb_system")
+            elif self.install_layout.lower() in ['unix']:
+                self.select_scheme("posix_prefix")
+            else:
+                raise DistutilsOptionError(
+                    "unknown value for --install-layout")
+        elif ((self.prefix_option and
+               os.path.normpath(self.prefix) != '/usr/local')
+              or sys.base_prefix != sys.prefix
+              or 'PYTHONUSERBASE' in os.environ
+              or 'VIRTUAL_ENV' in os.environ
+              or 'real_prefix' in sys.__dict__):
+            self.select_scheme("posix_prefix")
+        else:
+            if os.path.normpath(self.prefix) == '/usr/local':
+                self.prefix = self.exec_prefix = '/usr'
+                self.install_base = self.install_platbase = '/usr'
+            self.select_scheme("posix_local")
+
+
+class install_egg_info(orig_install_egg_info.install_egg_info):
+    user_options = list(orig_install_egg_info.install_egg_info.user_options) + [
+        ('install-layout', None, "custom installation layout"),
+    ]
+
+    def initialize_options(self):
+        super().initialize_options()
+        self.prefix_option = None
+        self.install_layout = None
+
+    def finalize_options(self):
+        self.set_undefined_options('install',('install_layout','install_layout'))
+        self.set_undefined_options('install',('prefix_option','prefix_option'))
+        super().finalize_options()
+
+    @property
+    def basename(self):
+        if self.install_layout:
+            if not self.install_layout.lower() in ['deb', 'unix']:
+                raise DistutilsOptionError(
+                    "unknown value for --install-layout")
+            no_pyver = (self.install_layout.lower() == 'deb')
+        elif self.prefix_option:
+            no_pyver = False
+        else:
+            no_pyver = True
+        if no_pyver:
+            basename = "%s-%s.egg-info" % (
+                to_filename(safe_name(self.distribution.get_name())),
+                to_filename(safe_version(self.distribution.get_version()))
+                )
+        else:
+            basename = "%s-%s-py%d.%d.egg-info" % (
+                to_filename(safe_name(self.distribution.get_name())),
+                to_filename(safe_version(self.distribution.get_version())),
+                *sys.version_info[:2]
+            )
+        return basename
+
+
+def _posix_lib(standard_lib, libpython, early_prefix, prefix):
+    is_default_prefix = not early_prefix or os.path.normpath(early_prefix) in ('/usr', '/usr/local')
+    if standard_lib:
+        return libpython
+    elif (is_default_prefix and
+              'PYTHONUSERBASE' not in os.environ and
+              'VIRTUAL_ENV' not in os.environ and
+              'real_prefix' not in sys.__dict__ and
+              sys.prefix == sys.base_prefix):
+            return os.path.join(prefix, "lib", "python3", "dist-packages")
+    else:
+        return os.path.join(libpython, "site-packages")
+
+
+def _inject_headers(name, scheme):
+    """
+    Given a scheme name and the resolved scheme,
+    if the scheme does not include headers, resolve
+    the fallback scheme for the name and use headers
+    from it. pypa/distutils#88
+    """
+    # Bypass the preferred scheme, which may not
+    # have defined headers.
+    fallback = orig_install._load_scheme('posix_prefix')
+    scheme.setdefault('headers', fallback['headers'])
+    return scheme
+
+
+def apply_customizations():
+    orig_install.install = install
+    orig_install._inject_headers = _inject_headers
+    orig_install_egg_info.install_egg_info = install_egg_info
+    distutils.sysconfig._posix_lib = _posix_lib
+
+
+apply_customizations()
diff --git a/lib-python/3/distutils/command/install.py b/lib-python/3/distutils/command/install.py
index faa371a..ad99883 100644
--- a/lib-python/3/distutils/command/install.py
+++ b/lib-python/3/distutils/command/install.py
@@ -35,6 +35,20 @@ INSTALL_SCHEMES = {
         'scripts': '$base/bin',
         'data'   : '$base',
         },
+    'unix_local': {
+        'purelib': '$base/local/lib/$implementation_lower$py_version_short/dist-packages',
+        'platlib': '$base/local/lib/$implementation_lower$py_version_short/dist-packages',
+        'headers': '$base/local/include/$implementation_lower$py_version_short/$dist_name',
+        'scripts': '$base/local/bin',
+        'data'   : '$base/local',
+        },
+    'deb_system': {
+        'purelib': '$base/lib/python3/dist-packages',
+        'platlib': '$base/lib/python3/dist-packages',
+        'headers': '$base/include/$implementation_lower$py_version_short/$dist_name',
+        'scripts': '$base/bin',
+        'data'   : '$base',
+        },
     'unix_home': {
         'purelib': '$base/lib/$implementation_lower',
         'platlib': '$base/lib/$implementation_lower',
@@ -136,6 +150,9 @@ class install(Command):
 
         ('record=', None,
          "filename in which to record list of installed files"),
+
+        ('install-layout=', None,
+         "installation layout to choose (known values: deb, unix)"),
         ]
 
     boolean_options = ['compile', 'force', 'skip-build']
@@ -156,6 +173,7 @@ class install(Command):
         self.exec_prefix = None
         self.home = None
         self.user = 0
+        self.prefix_option = None
 
         # These select only the installation base; it's up to the user to
         # specify the installation scheme (currently, that means supplying
@@ -177,6 +195,9 @@ class install(Command):
         self.install_userbase = USER_BASE
         self.install_usersite = USER_SITE
 
+        # enable custom installation, known values: deb
+        self.install_layout = None
+
         self.compile = None
         self.optimize = None
 
@@ -420,6 +441,7 @@ class install(Command):
             self.install_base = self.install_platbase = self.home
             self.select_scheme("unix_home")
         else:
+            self.prefix_option = self.prefix
             if self.prefix is None:
                 if self.exec_prefix is not None:
                     raise DistutilsOptionError(
@@ -434,7 +456,26 @@ class install(Command):
 
             self.install_base = self.prefix
             self.install_platbase = self.exec_prefix
-            self.select_scheme("unix_prefix")
+            if self.install_layout:
+                if self.install_layout.lower() in ['deb']:
+                    self.select_scheme("deb_system")
+                elif self.install_layout.lower() in ['posix', 'unix']:
+                    self.select_scheme("unix_prefix")
+                else:
+                    raise DistutilsOptionError(
+                            "unknown value for --install-layout")
+            elif ((self.prefix_option
+                        and not os.path.normpath(self.prefix).startswith(
+                            '/usr/local/'))
+                    or 'PYTHONUSERBASE' in os.environ
+                    or sys.prefix != sys.base_prefix
+                    or 'real_prefix' in sys.__dict__):
+                self.select_scheme("unix_prefix")
+            else:
+                if os.path.normpath(self.prefix).startswith('/usr/local/'):
+                    self.select_scheme("deb_system")
+                else:
+                    self.select_scheme("unix_local")
 
     def finalize_other(self):
         """Finalizes options for non-posix platforms"""
diff --git a/lib-python/3/distutils/command/install_egg_info.py b/lib-python/3/distutils/command/install_egg_info.py
index 0ddc736..0a71b61 100644
--- a/lib-python/3/distutils/command/install_egg_info.py
+++ b/lib-python/3/distutils/command/install_egg_info.py
@@ -14,18 +14,38 @@ class install_egg_info(Command):
     description = "Install package's PKG-INFO metadata as an .egg-info file"
     user_options = [
         ('install-dir=', 'd', "directory to install to"),
+        ('install-layout', None, "custom installation layout"),
     ]
 
     def initialize_options(self):
         self.install_dir = None
+        self.install_layout = None
+        self.prefix_option = None
 
     def finalize_options(self):
         self.set_undefined_options('install_lib',('install_dir','install_dir'))
-        basename = "%s-%s-py%d.%d.egg-info" % (
-            to_filename(safe_name(self.distribution.get_name())),
-            to_filename(safe_version(self.distribution.get_version())),
-            *sys.version_info[:2]
-        )
+        self.set_undefined_options('install',('install_layout','install_layout'))
+        self.set_undefined_options('install',('prefix_option','prefix_option'))
+        if self.install_layout:
+            if not self.install_layout.lower() in ['deb', 'unix']:
+                raise DistutilsOptionError(
+                    "unknown value for --install-layout")
+            no_pyver = (self.install_layout.lower() == 'deb')
+        elif self.prefix_option:
+            no_pyver = False
+        else:
+            no_pyver = True
+        if no_pyver:
+            basename = "%s-%s.egg-info" % (
+                to_filename(safe_name(self.distribution.get_name())),
+                to_filename(safe_version(self.distribution.get_version()))
+                )
+        else:
+            basename = "%s-%s-py%d.%d.egg-info" % (
+                to_filename(safe_name(self.distribution.get_name())),
+                to_filename(safe_version(self.distribution.get_version())),
+                *sys.version_info[:2]
+            )
         self.target = os.path.join(self.install_dir, basename)
         self.outputs = [self.target]
 
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
index 7318675..59f2e5b 100644
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -137,6 +137,8 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
     If 'prefix' is supplied, use it instead of sys.base_prefix or
     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
     """
+    is_default_prefix = not prefix or os.path.normpath(prefix) == '/usr'
+    in_ve = 'real_prefix' in sys.__dict__ or sys.prefix != sys.base_prefix
     if prefix is None:
         if standard_lib:
             prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
@@ -149,7 +151,10 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
         if standard_lib:
             return libpython
         else:
-            return os.path.join(libpython, "site-packages")
+            if is_default_prefix and 'PYTHONUSERBASE' not in os.environ and not in_ve:
+                return os.path.join(libpython, 'dist-packages')
+            else:
+                return os.path.join(libpython, 'site-packages')
     elif os.name == "nt":
         if standard_lib:
             return os.path.join(prefix, "Lib")
diff --git a/lib-python/3/pydoc.py b/lib-python/3/pydoc.py
index fe4e230..8f6f6d1 100644
--- a/lib-python/3/pydoc.py
+++ b/lib-python/3/pydoc.py
@@ -409,6 +409,7 @@ class Doc:
                                  'marshal', 'posix', 'signal', 'sys',
                                  '_thread', 'zipimport') or
              (file.startswith(basedir) and
+              not file.startswith(os.path.join(basedir, 'dist-packages')) and
               not file.startswith(os.path.join(basedir, 'site-packages')))) and
             object.__name__ not in ('xml.etree', 'test.pydoc_mod')):
             if docloc.startswith(("http://", "https://")):
diff --git a/lib-python/3/site.py b/lib-python/3/site.py
index 0160384..64c09c3 100644
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -6,13 +6,18 @@
 
 This will append site-specific paths to the module search path.  On
 Unix (including Mac OSX), it starts with sys.prefix and
-sys.exec_prefix (if different) and appends
-lib/python<version>/site-packages.
+sys.exec_prefix (if different) and appends dist-packages.
 On other platforms (such as Windows), it tries each of the
 prefixes directly, as well as with lib/site-packages appended.  The
 resulting directories, if they exist, are appended to sys.path, and
 also inspected for path configuration files.
 
+For Debian and derivatives, this sys.path is augmented with directories
+for packages distributed within the distribution. Local addons go
+into /usr/local/lib/pypy<version>/dist-packages, Debian addons
+install into /usr/lib/python3/dist-packages (which is shared with cPython).
+/usr/lib/pypy<version>/site-packages is not used.
+
 If a file named "pyvenv.cfg" exists one directory above sys.executable,
 sys.prefix and sys.exec_prefix are set to that directory and
 it is also checked for site-packages (sys.base_prefix and
@@ -336,6 +341,7 @@ def getsitepackages(prefixes=None):
     """
     sitepackages = []
     seen = set()
+    in_ve = 'real_prefix' in sys.__dict__ or sys.prefix != sys.base_prefix
 
     if prefixes is None:
         prefixes = PREFIXES
@@ -348,9 +354,15 @@ def getsitepackages(prefixes=None):
         implementation = _get_implementation().lower()
         ver = sys.version_info
         if os.sep == '/':
-            sitepackages.append(os.path.join(prefix, "lib",
-                                        f"{implementation}{ver[0]}.{ver[1]}",
-                                        "site-packages"))
+            if in_ve:
+                sitepackages.append(os.path.join(
+                    prefix, "lib", f"{implementation}{ver[0]}.{ver[1]}",
+                    "site-packages"))
+            sitepackages.append(os.path.join(
+                prefix, "local", "lib",
+                f"{implementation}{ver[0]}.{ver[1]}", "dist-packages"))
+            sitepackages.append(os.path.join(
+                prefix, "lib", "python3", "dist-packages"))
         else:
             sitepackages.append(prefix)
             sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
index ca073d1..feeb6c2 100644
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -46,6 +46,30 @@ _INSTALL_SCHEMES = {
         'scripts': '{base}/bin',
         'data': '{base}',
         },
+    'posix_local': {
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/local/lib/{implementation_lower}{py_version_short}/dist-packages',
+        'platlib': '{platbase}/local/lib/{implementation_lower}{py_version_short}/dist-packages',
+        'include':
+            '{installed_base}/local/include/{implementation_lower}{py_version_short}{abiflags}',
+        'platinclude':
+            '{installed_platbase}/local/include/{implementation_lower}{py_version_short}{abiflags}',
+        'scripts': '{base}/local/bin',
+        'data': '{base}',
+        },
+    'deb_system': {
+        'stdlib': '{installed_base}/lib/implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/python3/dist-packages',
+        'platlib': '{platbase}/lib/python3/dist-packages',
+        'include':
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
+        'platinclude':
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
+        'scripts': '{base}/bin',
+        'data': '{base}',
+        },
     'nt': {
         'stdlib': '{installed_base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -154,7 +178,7 @@ def is_python_build(check_home=False):
 _PYTHON_BUILD = is_python_build(True)
 
 if _PYTHON_BUILD:
-    for scheme in ('posix_prefix', 'posix_home'):
+    for scheme in ('posix_prefix', 'posix_home', 'posix_local', 'deb_system'):
         _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
         _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
 
@@ -191,8 +215,16 @@ def _expand_vars(scheme, vars):
 
 def _get_default_scheme():
     if os.name == 'posix':
-        # the default scheme for posix is posix_prefix
-        return 'posix_prefix'
+        if 'real_prefix' in sys.__dict__ or 'VIRTUAL_ENV' in os.environ:
+            # virtual environments
+            return 'posix_prefix'
+        else:
+            # default to /usr for package builds, /usr/local otherwise
+            deb_build = os.environ.get('DEB_PYTHON_INSTALL_LAYOUT', 'posix_local')
+            if deb_build in ('deb', 'deb_system'):
+                return 'deb_system'
+            else:
+                return 'posix_local'
     return os.name
 
 
@@ -515,7 +547,7 @@ def get_config_h_filename():
         else:
             inc_dir = _sys_home or _PROJECT_BASE
     else:
-        inc_dir = get_path('platinclude')
+        inc_dir = get_path('platinclude', 'posix_prefix')
     return os.path.join(inc_dir, 'pyconfig.h')
 
 
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
index b7e7b51..76d0e48 100644
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -229,8 +229,8 @@ class TestSysConfig(unittest.TestCase):
         self.assertTrue(os.path.isfile(config_h), config_h)
 
     def test_get_scheme_names(self):
-        wanted = ('nt', 'nt_user', 'osx_framework_user', 'posix_home',
-                  'posix_prefix', 'posix_user')
+        wanted = ('deb_system', 'nt', 'nt_user', 'osx_framework_user',
+                  'posix_home', 'posix_local', 'posix_prefix', 'posix_user')
         self.assertEqual(get_scheme_names(), wanted)
 
     @skip_unless_symlink
