diff --git a/autotest/gdrivers/data/zarr/kerchunk_parquet/parquet_ref_2_dim/ar/4.bin b/autotest/gdrivers/data/zarr/kerchunk_parquet/parquet_ref_2_dim/ar/4.bin
new file mode 100644
index 000000000000..ab2c6846789c
Binary files /dev/null and b/autotest/gdrivers/data/zarr/kerchunk_parquet/parquet_ref_2_dim/ar/4.bin differ
diff --git a/autotest/gdrivers/zarr_kerchunk_json.py b/autotest/gdrivers/zarr_kerchunk_json.py
index 21ec0674c97a..8f9cf5ed0fdd 100755
--- a/autotest/gdrivers/zarr_kerchunk_json.py
+++ b/autotest/gdrivers/zarr_kerchunk_json.py
@@ -245,16 +245,14 @@ def test_zarr_kerchunk_json_fail_exception(
         """{".zgroup": {"zarr_format":2}, "foo":["https://localhost:1/",0,1]}""",
     ],
 )
-def test_zarr_kerchunk_json_open_fail_no_exception(tmp_vsimem, content):
+def test_zarr_kerchunk_json_open_fail(tmp_vsimem, content):
 
     json_filename = str(tmp_vsimem / "test.json")
     gdal.FileFromMemBuffer(json_filename, content)
 
-    with gdal.quiet_errors():
-        assert (
-            gdal.VSIFOpenL("/vsikerchunk_json_ref/{" + json_filename + "}/foo", "rb")
-            is None
-        )
+    with pytest.raises(Exception):
+        with gdal.VSIFile("/vsikerchunk_json_ref/{" + json_filename + "}/foo", "rb"):
+            pass
 
 
 ###############################################################################
diff --git a/autotest/gdrivers/zarr_kerchunk_parquet.py b/autotest/gdrivers/zarr_kerchunk_parquet.py
index 7c843856981c..a50729cf7633 100755
--- a/autotest/gdrivers/zarr_kerchunk_parquet.py
+++ b/autotest/gdrivers/zarr_kerchunk_parquet.py
@@ -11,6 +11,10 @@
 # SPDX-License-Identifier: MIT
 ###############################################################################
 
+import os
+import shutil
+import sys
+
 import pytest
 
 from osgeo import gdal
@@ -318,6 +322,27 @@ def test_zarr_kerchunk_parquet_gdal_open_dim2():
 ###############################################################################
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Fails for some reason on Windows")
+def test_zarr_kerchunk_parquet_gdal_open_missing_bin_file(tmp_path):
+
+    shutil.copytree(
+        "data/zarr/kerchunk_parquet/parquet_ref_2_dim", tmp_path / "parquet_ref_2_dim"
+    )
+    os.unlink(tmp_path / "parquet_ref_2_dim/ar/4.bin")
+
+    with gdal.OpenEx(
+        f"ZARR:/vsikerchunk_parquet_ref/{{{tmp_path}/parquet_ref_2_dim}}",
+        gdal.OF_MULTIDIM_RASTER,
+    ) as ds:
+        rg = ds.GetRootGroup()
+        ar = rg.OpenMDArray("ar")
+        with pytest.raises(Exception):
+            ar.Read()
+
+
+###############################################################################
+
+
 @pytest.mark.require_driver("PARQUET")
 def test_zarr_kerchunk_parquet_invalid_parquet_struct():
     with gdal.OpenEx(
diff --git a/frmts/zarr/vsikerchunk_json_ref.cpp b/frmts/zarr/vsikerchunk_json_ref.cpp
index cd1f5bbaef9a..76e043fa4eda 100644
--- a/frmts/zarr/vsikerchunk_json_ref.cpp
+++ b/frmts/zarr/vsikerchunk_json_ref.cpp
@@ -21,6 +21,7 @@
 #include "cpl_json_streaming_parser.h"
 #include "cpl_json_streaming_writer.h"
 #include "cpl_mem_cache.h"
+#include "cpl_vsi_error.h"
 #include "cpl_vsi_virtual.h"
 
 #include "gdal_priv.h"
@@ -1583,7 +1584,14 @@ VSIKerchunkJSONRefFileSystem::Open(const char *pszFilename,
                      osPath.c_str());
         CPLConfigOptionSetter oSetter("GDAL_DISABLE_READDIR_ON_OPEN",
                                       "EMPTY_DIR", false);
-        return VSIFilesystemHandler::OpenStatic(osPath.c_str(), "rb");
+        auto fp = VSIFilesystemHandler::OpenStatic(osPath.c_str(), "rb", true);
+        if (!fp)
+        {
+            if (!VSIToCPLError(CE_Failure, CPLE_FileIO))
+                CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
+                         osPath.c_str());
+        }
+        return fp;
     }
 }
 
diff --git a/frmts/zarr/vsikerchunk_parquet_ref.cpp b/frmts/zarr/vsikerchunk_parquet_ref.cpp
index c8d25173890a..c7365760c79b 100644
--- a/frmts/zarr/vsikerchunk_parquet_ref.cpp
+++ b/frmts/zarr/vsikerchunk_parquet_ref.cpp
@@ -15,6 +15,7 @@
 
 #include "cpl_json.h"
 #include "cpl_mem_cache.h"
+#include "cpl_vsi_error.h"
 #include "cpl_vsi_virtual.h"
 
 #include "gdal_priv.h"
@@ -552,7 +553,15 @@ VSIVirtualHandleUniquePtr VSIKerchunkParquetRefFileSystem::Open(
                              osPath.c_str());
                 CPLConfigOptionSetter oSetter("GDAL_DISABLE_READDIR_ON_OPEN",
                                               "EMPTY_DIR", false);
-                return VSIFilesystemHandler::OpenStatic(osPath.c_str(), "rb");
+                auto fp = VSIFilesystemHandler::OpenStatic(osPath.c_str(), "rb",
+                                                           true);
+                if (!fp)
+                {
+                    if (!VSIToCPLError(CE_Failure, CPLE_FileIO))
+                        CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
+                                 osPath.c_str());
+                }
+                return fp;
             }
         }
 
diff --git a/port/cpl_vsil_subfile.cpp b/port/cpl_vsil_subfile.cpp
index fb5fcd0cdfc4..1c8d037c1c60 100644
--- a/port/cpl_vsil_subfile.cpp
+++ b/port/cpl_vsil_subfile.cpp
@@ -337,8 +337,8 @@ int VSISubFileFilesystemHandler::DecomposePath(const char *pszPath,
 
 VSIVirtualHandleUniquePtr
 VSISubFileFilesystemHandler::Open(const char *pszFilename,
-                                  const char *pszAccess, bool /* bSetError */,
-                                  CSLConstList /* papszOptions */)
+                                  const char *pszAccess, bool bSetError,
+                                  CSLConstList papszOptions)
 
 {
     if (!STARTS_WITH_CI(pszFilename, "/vsisubfile/"))
@@ -368,7 +368,8 @@ VSISubFileFilesystemHandler::Open(const char *pszFilename,
     /* -------------------------------------------------------------------- */
     /*      Open the underlying file.                                       */
     /* -------------------------------------------------------------------- */
-    auto fp = VSIFilesystemHandler::OpenStatic(osSubFilePath, pszAccess);
+    auto fp = VSIFilesystemHandler::OpenStatic(osSubFilePath, pszAccess,
+                                               bSetError, papszOptions);
 
     if (fp == nullptr)
         return nullptr;