TST: Make encoded sep check more locale sensitive (#14161)

Closes gh-14140.

Author: gfyoung

pandas/io/parsers.py

@@ -800,17 +800,22 @@ def _clean_options(self, options, engine):
                                   " different from '\s+' are"\
                                   " interpreted as regex)"
                 engine = 'python'
-
-        elif len(sep.encode(encoding)) > 1:
-            if engine not in ('python', 'python-fwf'):
-                fallback_reason = "the separator encoded in {encoding}"\
-                                  " is > 1 char long, and the 'c' engine"\
-                                  " does not support such separators".format(
-                                      encoding=encoding)
-                engine = 'python'
         elif delim_whitespace:
             if 'python' in engine:
                 result['delimiter'] = '\s+'
+        elif sep is not None:
+            encodeable = True
+            try:
+                if len(sep.encode(encoding)) > 1:
+                    encodeable = False
+            except UnicodeDecodeError:
+                encodeable = False
+            if not encodeable and engine not in ('python', 'python-fwf'):
+                fallback_reason = "the separator encoded in {encoding}" \
+                                  " is > 1 char long, and the 'c' engine" \
+                                  " does not support such separators".format(
+                                      encoding=encoding)
+                engine = 'python'
 
         if fallback_reason and engine_specified:
             raise ValueError(fallback_reason)

pandas/io/tests/parser/test_unsupported.py

@@ -60,10 +60,6 @@ def test_c_engine(self):
                        sep=None, delim_whitespace=False)
         with tm.assertRaisesRegexp(ValueError, msg):
             read_table(StringIO(data), engine='c', sep='\s')
-
-        # GH 14120, skipping as failing when locale is set
-        # with tm.assertRaisesRegexp(ValueError, msg):
-        #     read_table(StringIO(data), engine='c', sep='§')
         with tm.assertRaisesRegexp(ValueError, msg):
             read_table(StringIO(data), engine='c', skipfooter=1)