Defer code generator validation until transform submission

.github/workflows/ci.yaml

@@ -13,10 +13,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
 
@@ -35,7 +35,7 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6

.github/workflows/ci_production.yaml

@@ -11,10 +11,10 @@ jobs:
     environment: production-service
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up Python 3.12
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.12"
 

.github/workflows/docs.yml

@@ -20,12 +20,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.12'
 
@@ -49,7 +49,7 @@ jobs:
         done
 
     - name: Upload artifact
-      uses: actions/upload-pages-artifact@v3
+      uses: actions/upload-pages-artifact@v4
       with:
         path: 'docs/_build/html'
 

.github/workflows/pypi.yaml

@@ -11,17 +11,17 @@ jobs:
     # c.f. https://docs.pypi.org/trusted-publishers/using-a-publisher/
     permissions:
       id-token: write
-      contents: write # required to push with GITHUB_TOKEN
     env:
       BRANCH: ${{ github.event.release.target_commitish }}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         fetch-depth: 0
+        ssh-key: ${{ secrets.DEPLOY_KEY }}
 
     - name: Set up Python 3.12
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.12'
 
@@ -45,6 +45,24 @@ jobs:
       run: |
         sed -i "/version =/ s/= \"[^\"]*\"/= \"${{ env.RELEASE_VERSION }}\"/" pyproject.toml
 
+    - name: commit version-bump
+      run: |
+        git config --local user.name "github-actions[bot]"
+        git config --local user.email "github-actions[bot]@users.noreply.github.com"
+        git add ./pyproject.toml
+
+        # Check if there are changes to commit
+        if ! git diff --cached --quiet; then
+          echo "Changes detected, committing..."
+          git commit -m "Update version to ${{ env.RELEASE_VERSION }}" --no-verify
+        else
+          echo "No changes to commit"
+        fi
+
+    - name: Push commit
+      run: |
+        git push origin HEAD:$BRANCH
+
     - name: Build a sdist and wheel
       run: |
         python -m build .
@@ -66,6 +84,6 @@ jobs:
 
     - name: Publish distribution 📦 to PyPI
       if: github.repository == 'ssl-hep/ServiceX_frontend'
-      uses: pypa/gh-action-pypi-publish@v1.12.4
+      uses: pypa/gh-action-pypi-publish@v1.13.0
       with:
         print-hash: true

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v5.0.0"
+    rev: "v6.0.0"
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -15,7 +15,7 @@ repos:
     hooks:
       - id: flake8
   - repo: https://github.com/psf/black
-    rev: 25.1.0
+    rev: 25.9.0
     hooks:
       -   id: black
 

pyproject.toml

@@ -80,6 +80,7 @@ test = [
     "pyarrow>=12.0.0",
     "pre-commit>=4.0.1",
     "pytest-aioboto3>=0.6.0",
+    "coverage>=7.0.0",
 ]
 docs = [
     "sphinx>=7.0.1, <8.2.0",
@@ -124,14 +125,20 @@ include = [
 packages = ["servicex"]
 
 [tool.coverage.run]
-dynamic_context = "test_function"
+source = ["servicex"]
+omit = [
+    "*/tests/*",
+    "*/test_*",
+]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+testpaths = ["tests"]
 
 [tool.hatch.envs.test]
 features = ["test"]
 
 [tool.hatch.envs.test.scripts]
 test = "pytest {args}"
-cov = "pytest --cov=servicex {args}"
+cov = "coverage run --source servicex/ -m pytest tests && coverage report"
+cov-html = "coverage run --source servicex/ -m pytest tests && coverage html"

servicex/app/cache.py

@@ -29,11 +29,29 @@
 
 import rich
 import typer
+from pathlib import Path
 from rich.prompt import Confirm
+from typing import List
 
 from servicex.app import pipeable_table
+from servicex.models import TransformedResults
 from servicex.servicex_client import ServiceXClient
 
+
+def _format_size(size_bytes: int) -> str:
+    """Return human readable string for size in bytes."""
+    if size_bytes >= 1024**4:
+        size = size_bytes / (1024**4)
+        unit = "TB"
+    elif size_bytes >= 1024**3:
+        size = size_bytes / (1024**3)
+        unit = "GB"
+    else:
+        size = size_bytes / (1024**2)
+        unit = "MB"
+    return f"{size:,.2f} {unit}"
+
+
 cache_app = typer.Typer(name="cache", no_args_is_help=True)
 force_opt = typer.Option(False, "-y", help="Force, don't ask for permission")
 transform_id_arg = typer.Argument(help="Transform ID")
@@ -48,7 +66,9 @@ def cache():
 
 
 @cache_app.command()
-def list():
+def list(
+    show_size: bool = typer.Option(False, "--size", help="Include size of cached files")
+) -> None:
     """
     List the cached queries
     """
@@ -61,16 +81,26 @@ def list():
     table.add_column("Run Date")
     table.add_column("Files")
     table.add_column("Format")
-    runs = cache.cached_queries()
+    if show_size:
+        table.add_column("Size")
+
+    runs: List[TransformedResults] = cache.cached_queries()
     for r in runs:
-        table.add_row(
+        row = [
             r.title,
             r.codegen,
             r.request_id,
             r.submit_time.astimezone().strftime("%a, %Y-%m-%d %H:%M"),
             str(r.files),
             r.result_format,
-        )
+        ]
+        if show_size:
+            total_size: int = sum(
+                Path(f).stat().st_size for f in r.file_list if Path(f).exists()
+            )
+            # Convert to human readable string, keeping two decimal places
+            row.append(_format_size(total_size))
+        table.add_row(*row)
     rich.print(table)
 
 

servicex/app/codegen.py

@@ -36,6 +36,12 @@
 codegen_app = typer.Typer(name="codegen", no_args_is_help=True)
 
 
+@codegen_app.callback()
+def codegen() -> None:
+    """Sub-commands for interacting with available backend code generators."""
+    pass
+
+
 @codegen_app.command(no_args_is_help=False)
 def list(
     backend: Optional[str] = backend_cli_option,

servicex/app/datasets.py

@@ -25,7 +25,8 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-from typing import Optional
+from fnmatch import fnmatch
+from typing import List, Optional
 
 import rich
 
@@ -35,9 +36,18 @@
 import typer
 
 from servicex.servicex_client import ServiceXClient
+from servicex.models import CachedDataset
 from rich.table import Table
 
 datasets_app = typer.Typer(name="datasets", no_args_is_help=True)
+
+
+@datasets_app.callback()
+def datasets() -> None:
+    """Sub-commands for interacting with the list of looked-up datasets on the server."""
+    pass
+
+
 did_finder_opt = typer.Option(
     None,
     help="Filter datasets by DID finder. Some useful values are 'rucio' or 'user'",
@@ -49,18 +59,24 @@
     show_default=True,
 )
 dataset_id_get_arg = typer.Argument(..., help="The ID of the dataset to get")
-dataset_id_delete_arg = typer.Argument(..., help="The ID of the dataset to delete")
+dataset_ids_delete_arg = typer.Argument(..., help="IDs of the datasets to delete")
 
 
 @datasets_app.command(no_args_is_help=False)
 def list(
+    name_pattern: Optional[str] = typer.Argument(
+        None,
+        help="Filter datasets by name. Use '*' as a wildcard for any number of characters.",
+    ),
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     did_finder: Optional[str] = did_finder_opt,
     show_deleted: Optional[bool] = show_deleted_opt,
-):
+) -> None:
     """
-    List the datasets. Use fancy formatting if printing to a terminal.
+    List the datasets on the server.
+
+    Use fancy formatting if printing to a terminal.
     Output as plain text if redirected.
     """
     sx = ServiceXClient(backend=backend, config_path=config_path)
@@ -74,7 +90,24 @@ def list(
     if show_deleted:
         table.add_column("Deleted")
 
-    datasets = sx.get_datasets(did_finder=did_finder, show_deleted=show_deleted)
+    datasets: List[CachedDataset] = sx.get_datasets(
+        did_finder=did_finder, show_deleted=show_deleted
+    )
+
+    if name_pattern:
+        # Allow substring matching when no wildcard is provided by surrounding the pattern
+        # with '*' characters. Users can still provide wildcards explicitly to narrow the match.
+        effective_pattern = name_pattern if "*" in name_pattern else f"*{name_pattern}*"
+
+        def matches_pattern(dataset: CachedDataset) -> bool:
+            display_name = dataset.name if dataset.did_finder != "user" else "File list"
+            return any(
+                fnmatch(candidate, effective_pattern)
+                for candidate in {dataset.name, display_name}
+            )
+
+        datasets = [dataset for dataset in datasets if matches_pattern(dataset)]
+    assert show_deleted is not None
 
     for d in datasets:
         # Format the CachedDataset object into a table row
@@ -85,11 +118,25 @@ def list(
         d_name = d.name if d.did_finder != "user" else "File list"
         is_stale = "Yes" if d.is_stale else ""
         last_used = d.last_used.strftime("%Y-%m-%dT%H:%M:%S")
+
+        # Convert byte size into a human-readable string with appropriate units
+        size_in_bytes = d.size
+        if size_in_bytes >= 1e12:
+            size_value = size_in_bytes / 1e12
+            unit = "TB"
+        elif size_in_bytes >= 1e9:
+            size_value = size_in_bytes / 1e9
+            unit = "GB"
+        else:
+            size_value = size_in_bytes / 1e6
+            unit = "MB"
+        size_str = f"{size_value:,.2f} {unit}"
+
         table.add_row(
             str(d.id),
             d_name,
-            "%d" % d.n_files,
-            "{:,}MB".format(round(d.size / 1e6)),
+            f"{d.n_files}",
+            size_str,
             d.lookup_status,
             last_used,
             is_stale,
@@ -104,7 +151,11 @@ def get(
     dataset_id: int = dataset_id_get_arg,
 ):
     """
-    Get the details of a dataset. Output as a pretty, nested table if printing to a terminal.
+    List the files in a dataset.
+
+    Known replicas on the GRID are listed.
+
+    Output as a pretty, nested table if printing to a terminal.
     Output as json if redirected.
     """
     sx = ServiceXClient(backend=backend, config_path=config_path)
@@ -145,12 +196,22 @@ def get(
 def delete(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
-    dataset_id: int = dataset_id_delete_arg,
+    dataset_ids: List[int] = dataset_ids_delete_arg,
 ):
+    """
+    Remove a dataset from the ServiceX.
+
+    The next time it is queried, it will have to be looked up again. This command should only be
+    used when debugging.
+    """
     sx = ServiceXClient(backend=backend, config_path=config_path)
-    result = sx.delete_dataset(dataset_id)
-    if result:
-        typer.echo(f"Dataset {dataset_id} deleted")
-    else:
-        typer.echo(f"Dataset {dataset_id} not found")
+    any_missing: bool = False  # Track if any dataset ID is not found
+    for dataset_id in dataset_ids:
+        result = sx.delete_dataset(dataset_id)
+        if result:
+            typer.echo(f"Dataset {dataset_id} deleted")
+        else:
+            typer.echo(f"Dataset {dataset_id} not found")
+            any_missing = True
+    if any_missing:
         raise typer.Abort()