gh-37024: Access database and other files through features, for simpler configuration
    
This PR reworks the way sagemath access database files, and some other
external files. The end result is that the runtime detection of
databases as features matches exactly what will be used. Moreover, this
leads to an easy implementation of search paths, meaning we don't need
configuration as long as the files are in a few standard system
locations.

Notes on implementation:
- First every user of these files access them via features, and never
directly through `sage.env`, since the former is dynamic and the latter
is static.
- Then we add a variable `SAGE_DATA_PATH` which is a colon separated
search path for databases. The default should work for sage-the-distro
and most distros (think `/usr/share/sagemath:/usr/share` for a system
install on `/usr`).
- Now a database, say `cremona` is searched on `$p/cremona` for each
`$p` in `SAGE_DATA_PATH`.
- I also added `$DOT_SAGE/db` first in the search path, this makes it
easy for a user to install a missing database (or update an old one).
- Finally, as an example I did something similar to find `JmolData.jar`
and `three.min.js`. The former is searched in
`$SAGE_SHARE/sagemath/jmol` and `$SAGE_SHARE/jmol` and the latter also
in `$SAGE_SHARE/jupyter/nbextensions/threejs-sage` which seems to make
sense.

Other files can be done later (e.g. mathjax). In principle anything that
has a default value in `sage.env` or `sage_conf` should benefit from
using a search path and removing the default value.

With this PR, I can run stock sagemath without any `sage_conf.py`.

### 📝 Checklist

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have created tests covering the changes.

### ⌛ Dependencies
    
URL: #37024
Reported by: Gonzalo Tornaría
Reviewer(s): François Bissey, Gonzalo Tornaría, Matthias Köppe, Michael Orlitzky

Author: Release Manager

src/sage/combinat/designs/MOLS_handbook_data.py

@@ -75,7 +75,7 @@
       0|                                                           +               +
      20|
      40|
-     60|   +
+     60|
      80|
     100|
     120|
@@ -126,7 +126,6 @@
 from sage.rings.integer import Integer
 from sage.categories.sets_cat import EmptySetError
 from sage.misc.unknown import Unknown
-from sage.env import COMBINATORIAL_DESIGN_DATA_DIR
 
 
 def are_mutually_orthogonal_latin_squares(l, verbose=False):
@@ -500,13 +499,13 @@ def MOLS_table(start,stop=None,compare=False,width=None):
           0|                                                           +               +
          20|
          40|
-         60|   +
+         60|
          80|
         sage: MOLS_table(50, 100, compare=True)
                0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
             ________________________________________________________________________________
          40|
-         60|   +
+         60|
          80|
     """
     from .orthogonal_arrays import largest_available_k
@@ -520,11 +519,6 @@ def MOLS_table(start,stop=None,compare=False,width=None):
     if stop <= start:
         return
 
-    if compare:
-        handbook_file = open("{}/MOLS_table.txt".format(COMBINATORIAL_DESIGN_DATA_DIR), 'r')
-        hb = [int(_) for _ in handbook_file.readlines()[9].split(',')]
-        handbook_file.close()
-
     # choose an appropriate width (needs to be >= 3 because "+oo" should fit)
     if width is None:
         width = max(3, Integer(stop-1).ndigits(10))
@@ -537,9 +531,11 @@ def MOLS_table(start,stop=None,compare=False,width=None):
             print("\n{:>{width}}|".format(i, width=width), end="")
         k = largest_available_k(i)-2
         if compare:
-            if i < 2 or hb[i] == k:
+            from . import MOLS_handbook_data
+            lower_bound = MOLS_handbook_data.lower_bound(i)
+            if i < 2 or lower_bound == k:
                 c = ""
-            elif hb[i] < k:
+            elif lower_bound < k:
                 c = "+"
             else:
                 c = "-"

src/sage/combinat/designs/latin_squares.py

@@ -79,8 +79,6 @@
 
 from sage.features.databases import DatabaseJones
 
-JONESDATA = os.path.join(SAGE_SHARE, 'jones')  # should match the filename set in DatabaseJones
-
 
 def sortkey(K):
     """
@@ -160,8 +158,10 @@ def _init(self, path):
                 for Y in os.listdir(Z):
                     if Y[-3:] == ".gp":
                         self._load(Z, Y)
-        os.makedirs(JONESDATA, exist_ok=True)
-        save(self.root, JONESDATA + "/jones.sobj")
+
+        data_dir = os.path.dirname(DatabaseJones().absolute_filename())
+        os.makedirs(data_dir, exist_ok=True)
+        save(self.root, os.path.join(data_dir, "jones.sobj"))
 
     def unramified_outside(self, S, d=None, var='a'):
         """

src/sage/databases/jones.py

@@ -250,7 +250,6 @@ def construct_skeleton(database):
     skeleton = {}
     cur = database.__connection__.cursor()
     exe = cur.execute("SELECT name FROM sqlite_master WHERE TYPE='table'")
-    from sage.env import GRAPHS_DATA_DIR
     for table in exe.fetchall():
         skeleton[table[0]] = {}
         exe1 = cur.execute("PRAGMA table_info(%s)" % table[0])
@@ -264,8 +263,7 @@ def construct_skeleton(database):
         exe2 = cur.execute("PRAGMA index_list(%s)" % table[0])
         for col in exe2.fetchall():
             if col[1].find('sqlite') == -1:
-                if database.__dblocation__ == \
-                        os.path.join(GRAPHS_DATA_DIR,'graphs.db'):
+                if os.path.basename(database.__dblocation__) == 'graphs.db':
                     name = col[1]
                 else:
                     name = col[1][len(table[0])+3:]

src/sage/databases/sql_db.py

@@ -195,18 +195,26 @@ def var(key: str, *fallbacks: Optional[str], force: bool = False) -> Optional[st
 SAGE_ARCHFLAGS = var("SAGE_ARCHFLAGS", "unset")
 SAGE_PKG_CONFIG_PATH = var("SAGE_PKG_CONFIG_PATH")
 
+# colon-separated search path for databases.
+SAGE_DATA_PATH = var("SAGE_DATA_PATH",
+                     os.pathsep.join(filter(None, [
+                         join(DOT_SAGE, "db"),
+                         join(SAGE_SHARE, "sagemath"),
+                         SAGE_SHARE,
+                         ])))
+
+# database directories, the default is to search in SAGE_DATA_PATH
+CREMONA_LARGE_DATA_DIR = var("CREMONA_LARGE_DATA_DIR")
+CREMONA_MINI_DATA_DIR = var("CREMONA_MINI_DATA_DIR")
+ELLCURVE_DATA_DIR = var("ELLCURVE_DATA_DIR")
+GRAPHS_DATA_DIR = var("GRAPHS_DATA_DIR")
+POLYTOPE_DATA_DIR = var("POLYTOPE_DATA_DIR")
+
 # installation directories for various packages
-GRAPHS_DATA_DIR = var("GRAPHS_DATA_DIR", join(SAGE_SHARE, "graphs"))
-ELLCURVE_DATA_DIR = var("ELLCURVE_DATA_DIR", join(SAGE_SHARE, "ellcurves"))
-POLYTOPE_DATA_DIR = var("POLYTOPE_DATA_DIR", join(SAGE_SHARE, "reflexive_polytopes"))
-
-COMBINATORIAL_DESIGN_DATA_DIR = var("COMBINATORIAL_DESIGN_DATA_DIR", join(SAGE_SHARE, "combinatorial_designs"))
-CREMONA_MINI_DATA_DIR = var("CREMONA_MINI_DATA_DIR", join(SAGE_SHARE, "cremona"))
-CREMONA_LARGE_DATA_DIR = var("CREMONA_LARGE_DATA_DIR", join(SAGE_SHARE, "cremona"))
-JMOL_DIR = var("JMOL_DIR", join(SAGE_SHARE, "jmol"))
+JMOL_DIR = var("JMOL_DIR")
 MATHJAX_DIR = var("MATHJAX_DIR", join(SAGE_SHARE, "mathjax"))
 MTXLIB = var("MTXLIB", join(SAGE_SHARE, "meataxe"))
-THREEJS_DIR = var("THREEJS_DIR", join(SAGE_SHARE, "threejs-sage"))
+THREEJS_DIR = var("THREEJS_DIR")
 PPLPY_DOCS = var("PPLPY_DOCS", join(SAGE_SHARE, "doc", "pplpy"))
 MAXIMA = var("MAXIMA", "maxima")
 MAXIMA_FAS = var("MAXIMA_FAS")
@@ -313,6 +321,7 @@ def sage_include_directories(use_sources=False):
 
     return dirs
 
+
 def get_cblas_pc_module_name() -> str:
     """
     Return the name of the BLAS libraries to be used.
@@ -420,7 +429,7 @@ def cython_aliases(required_modules=None,
             aliases["ECL_INCDIR"] = list(map(lambda s: s[2:], filter(lambda s: s.startswith('-I'), ecl_cflags)))
             aliases["ECL_LIBDIR"] = list(map(lambda s: s[2:], filter(lambda s: s.startswith('-L'), ecl_libs)))
             aliases["ECL_LIBRARIES"] = list(map(lambda s: s[2:], filter(lambda s: s.startswith('-l'), ecl_libs)))
-            aliases["ECL_LIBEXTRA"] = list(filter(lambda s: not s.startswith(('-l','-L')), ecl_libs))
+            aliases["ECL_LIBEXTRA"] = list(filter(lambda s: not s.startswith(('-l', '-L')), ecl_libs))
             continue
         else:
             try:
@@ -439,7 +448,7 @@ def cython_aliases(required_modules=None,
         # include search order matters.
         aliases[var + "INCDIR"] = pc['include_dirs']
         aliases[var + "LIBDIR"] = pc['library_dirs']
-        aliases[var + "LIBEXTRA"] = list(filter(lambda s: not s.startswith(('-l','-L')), libs.split()))
+        aliases[var + "LIBEXTRA"] = list(filter(lambda s: not s.startswith(('-l', '-L')), libs.split()))
         aliases[var + "LIBRARIES"] = pc['libraries']
 
     # uname-specific flags

src/sage/env.py

@@ -416,6 +416,7 @@ def unhide(self):
         """
         self._hidden = False
 
+
 class FeatureNotPresentError(RuntimeError):
     r"""
     A missing feature error.
@@ -791,26 +792,37 @@ class StaticFile(FileFeature):
     EXAMPLES::
 
         sage: from sage.features import StaticFile
-        sage: StaticFile(name="no_such_file", filename="KaT1aihu", search_path=("/",), spkg="some_spkg", url="http://rand.om").require()  # optional - sage_spkg
+        sage: StaticFile(name="no_such_file", filename="KaT1aihu",              # optional - sage_spkg
+        ....:            search_path="/", spkg="some_spkg",
+        ....:            url="http://rand.om").require()
         Traceback (most recent call last):
         ...
         FeatureNotPresentError: no_such_file is not available.
         'KaT1aihu' not found in any of ['/']...
         To install no_such_file...you can try to run...sage -i some_spkg...
         Further installation instructions might be available at http://rand.om.
     """
-    def __init__(self, name, filename, search_path=None, **kwds):
+    def __init__(self, name, filename, *, search_path=None, **kwds):
         r"""
         TESTS::
 
             sage: from sage.features import StaticFile
-            sage: StaticFile(name="null", filename="null", search_path=("/dev",))
+            sage: StaticFile(name="null", filename="null", search_path="/dev")
             Feature('null')
+            sage: sh = StaticFile(name="shell", filename="sh",
+            ....:                 search_path=("/dev", "/bin", "/usr"))
+            sage: sh
+            Feature('shell')
+            sage: sh.absolute_filename()
+            '/bin/sh'
+
         """
         Feature.__init__(self, name, **kwds)
         self.filename = filename
         if search_path is None:
             self.search_path = [SAGE_SHARE]
+        elif isinstance(search_path, str):
+            self.search_path = [search_path]
         else:
             self.search_path = list(search_path)