[CI Problem] Not found: No algorithm worked in tensorflow tests #10275
Description
Some tensorflow frontend tests are intermittently failing:
test_forward_inception_v3in https://ci.tlcpack.ai/blue/organizations/jenkins/tvm/detail/main/2518/pipelinetest_forward_convolutionin https://ci.tlcpack.ai/blue/organizations/jenkins/tvm/detail/PR-10266/2/pipeline/
So these tests are disabled until a fix can be landed
Full error:
=================================== FAILURES ===================================
__________________________ test_forward_inception_v3 ___________________________
self = <tensorflow.python.client.session.Session object at 0x7f8bbc5b2ef0>
fn = <function BaseSession._do_run.<locals>._run_fn at 0x7f8bcc027d90>
args = ({<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8bbc532c00>: array([[[[0.3588705 , 0.7607201 , 0.11499...668 ]]]], dtype=float32)}, [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8b9c442e30>], [], None, None)
message = '2 root error(s) found.\n (0) Not found: No algorithm worked!\n\t [[node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchN...esting/tf.py:276) ]]\n\t [[InceptionV3/Predictions/Reshape_1/_3]]\n0 successful operations.\n0 derived errors ignored.'
m = <_sre.SRE_Match object; span=(63, 135), match='[[{{node InceptionV3/InceptionV3/Conv2d_1a_3x3/Ba>
def _do_call(self, fn, *args):
try:
> return fn(*args)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1375:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
feed_dict = {<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8bbc532c00>: array([[[[0.3588705 , 0.7607201 , 0.114994...9294],
[0.59303373, 0.2807652 , 0.59482074],
[0.2578382 , 0.7176947 , 0.7811668 ]]]], dtype=float32)}
fetch_list = [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8b9c442e30>]
target_list = [], options = None, run_metadata = None
def _run_fn(feed_dict, fetch_list, target_list, options, run_metadata):
# Ensure any changes to the graph are reflected in the runtime.
self._extend_graph()
return self._call_tf_sessionrun(options, feed_dict, fetch_list,
> target_list, run_metadata)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1360:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tensorflow.python.client.session.Session object at 0x7f8bbc5b2ef0>
options = None
feed_dict = {<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8bbc532c00>: array([[[[0.3588705 , 0.7607201 , 0.114994...9294],
[0.59303373, 0.2807652 , 0.59482074],
[0.2578382 , 0.7176947 , 0.7811668 ]]]], dtype=float32)}
fetch_list = [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8b9c442e30>]
target_list = [], run_metadata = None
def _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list,
run_metadata):
return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
fetch_list, target_list,
> run_metadata)
E tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
E (0) Not found: No algorithm worked!
E [[{{node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchNorm/batchnorm/mul}}]]
E (1) Not found: No algorithm worked!
E [[{{node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchNorm/batchnorm/mul}}]]
E [[InceptionV3/Predictions/Reshape_1/_3]]
E 0 successful operations.
E 0 derived errors ignored.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1453: NotFoundError
During handling of the above exception, another exception occurred:
def test_forward_inception_v3():
"""test inception V3 model"""
with tf.Graph().as_default():
graph_def = tf_testing.get_workload(
"InceptionV3/inception_v3_2016_08_28_frozen-with_shapes.pb"
)
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
data = np.random.uniform(size=(1, 299, 299, 3)).astype("float32")
with tf.Session() as sess:
> tf_output = run_tf_graph(sess, data, "input:0", "InceptionV3/Predictions/Reshape_1:0")
tests/python/frontend/tensorflow/test_forward.py:3784:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/python/frontend/tensorflow/test_forward.py:215: in run_tf_graph
output_data = sess.run(tensor, input_dict)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:968: in run
run_metadata_ptr)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1191: in _run
feed_dict_tensor, options, run_metadata)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1369: in _do_run
run_metadata)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tensorflow.python.client.session.Session object at 0x7f8bbc5b2ef0>
fn = <function BaseSession._do_run.<locals>._run_fn at 0x7f8bcc027d90>
args = ({<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8bbc532c00>: array([[[[0.3588705 , 0.7607201 , 0.11499...668 ]]]], dtype=float32)}, [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f8b9c442e30>], [], None, None)
message = '2 root error(s) found.\n (0) Not found: No algorithm worked!\n\t [[node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchN...esting/tf.py:276) ]]\n\t [[InceptionV3/Predictions/Reshape_1/_3]]\n0 successful operations.\n0 derived errors ignored.'
m = <_sre.SRE_Match object; span=(63, 135), match='[[{{node InceptionV3/InceptionV3/Conv2d_1a_3x3/Ba>
def _do_call(self, fn, *args):
try:
return fn(*args)
except errors.OpError as e:
message = compat.as_text(e.message)
m = BaseSession._NODEDEF_NAME_RE.search(message)
node_def = None
op = None
if m is not None:
node_name = m.group(3)
try:
op = self._graph.get_operation_by_name(node_name)
node_def = op.node_def
except KeyError:
pass
message = error_interpolation.interpolate(message, self._graph)
if 'only supports NHWC tensor format' in message:
message += ('\nA possible workaround: Try disabling Grappler optimizer'
'\nby modifying the config for creating the session eg.'
'\nsession_config.graph_options.rewrite_options.'
'disable_meta_optimizer = True')
> raise type(e)(node_def, op, message)
E tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
E (0) Not found: No algorithm worked!
E [[node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchNorm/batchnorm/mul (defined at workspace/python/tvm/relay/testing/tf.py:276) ]]
E (1) Not found: No algorithm worked!
E [[node InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchNorm/batchnorm/mul (defined at workspace/python/tvm/relay/testing/tf.py:276) ]]
E [[InceptionV3/Predictions/Reshape_1/_3]]
E 0 successful operations.
E 0 derived errors ignored.
E
E Original stack trace for 'InceptionV3/InceptionV3/Conv2d_1a_3x3/BatchNorm/batchnorm/mul':
E File "usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
E "__main__", mod_spec)
E File "usr/lib/python3.6/runpy.py", line 85, in _run_code
E exec(code, run_globals)
E File "usr/local/lib/python3.6/dist-packages/pytest/__main__.py", line 5, in <module>
E raise SystemExit(pytest.console_main())
E File "usr/local/lib/python3.6/dist-packages/_pytest/config/__init__.py", line 185, in console_main
E code = main()
E File "usr/local/lib/python3.6/dist-packages/_pytest/config/__init__.py", line 163, in main
E config=config
E File "usr/local/lib/python3.6/dist-packages/pluggy/_hooks.py", line 265, in __call__
E return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_manager.py", line 80, in _hookexec
E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_callers.py", line 39, in _multicall
E res = hook_impl.function(*args)
E File "usr/local/lib/python3.6/dist-packages/_pytest/main.py", line 316, in pytest_cmdline_main
E return wrap_session(config, _main)
E File "usr/local/lib/python3.6/dist-packages/_pytest/main.py", line 269, in wrap_session
E session.exitstatus = doit(config, session) or 0
E File "usr/local/lib/python3.6/dist-packages/_pytest/main.py", line 323, in _main
E config.hook.pytest_runtestloop(session=session)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_hooks.py", line 265, in __call__
E return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_manager.py", line 80, in _hookexec
E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_callers.py", line 39, in _multicall
E res = hook_impl.function(*args)
E File "usr/local/lib/python3.6/dist-packages/_pytest/main.py", line 348, in pytest_runtestloop
E item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_hooks.py", line 265, in __call__
E return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_manager.py", line 80, in _hookexec
E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_callers.py", line 39, in _multicall
E res = hook_impl.function(*args)
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 109, in pytest_runtest_protocol
E runtestprotocol(item, nextitem=nextitem)
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 126, in runtestprotocol
E reports.append(call_and_report(item, "call", log))
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 215, in call_and_report
E call = call_runtest_hook(item, when, **kwds)
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 255, in call_runtest_hook
E lambda: ihook(item=item, **kwds), when=when, reraise=reraise
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 311, in from_call
E result: Optional[TResult] = func()
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 255, in <lambda>
E lambda: ihook(item=item, **kwds), when=when, reraise=reraise
E File "usr/local/lib/python3.6/dist-packages/pluggy/_hooks.py", line 265, in __call__
E return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_manager.py", line 80, in _hookexec
E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_callers.py", line 39, in _multicall
E res = hook_impl.function(*args)
E File "usr/local/lib/python3.6/dist-packages/_pytest/runner.py", line 162, in pytest_runtest_call
E item.runtest()
E File "usr/local/lib/python3.6/dist-packages/_pytest/python.py", line 1641, in runtest
E self.ihook.pytest_pyfunc_call(pyfuncitem=self)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_hooks.py", line 265, in __call__
E return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_manager.py", line 80, in _hookexec
E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
E File "usr/local/lib/python3.6/dist-packages/pluggy/_callers.py", line 39, in _multicall
E res = hook_impl.function(*args)
E File "usr/local/lib/python3.6/dist-packages/_pytest/python.py", line 183, in pytest_pyfunc_call
E result = testfunction(**testargs)
E File "workspace/tests/python/frontend/tensorflow/test_forward.py", line 3776, in test_forward_inception_v3
E "InceptionV3/inception_v3_2016_08_28_frozen-with_shapes.pb"
E File "workspace/python/tvm/relay/testing/tf.py", line 276, in get_workload
E graph = tf_compat_v1.import_graph_def(graph_def, name="", input_map=inputs_dict)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py", line 538, in new_func
E return func(*args, **kwargs)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/importer.py", line 405, in import_graph_def
E producer_op_list=producer_op_list)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/importer.py", line 513, in _import_graph_def_internal
E _ProcessNewOps(graph)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/importer.py", line 243, in _ProcessNewOps
E for new_op in graph._add_new_tf_operations(compute_devices=False): # pylint: disable=protected-access
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 3680, in _add_new_tf_operations
E for c_op in c_api_util.new_tf_operations(self)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 3680, in <listcomp>
E for c_op in c_api_util.new_tf_operations(self)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 3561, in _create_op_from_tf_operation
E ret = Operation(c_op, self)
E File "usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 1990, in __init__
E self._traceback = tf_stack.extract_stack()
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1394: NotFoundError
------------------------------ Captured log call -------------------------------
WARNING tensorflow:deprecation.py:339 From /workspace/python/tvm/relay/testing/tf.py:273: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.gfile.GFile.
=============================== warnings summary ===============================
../usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py:22
/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py:22: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
../usr/lib/python3.6/importlib/_bootstrap.py:219
../usr/lib/python3.6/importlib/_bootstrap.py:219
/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject
return f(*args, **kwds)
-- Docs: https://docs.pytest.org/en/stable/warnings.html
- generated xml file: /workspace/build/pytest-results/python-frontend-tensorflow-cython.xml -
=========================== short test summary info ============================
FAILED tests/python/frontend/tensorflow/test_forward.py::test_forward_inception_v3
======================== 1 failed, 3 warnings in 10.72s ========================
script returned exit code 1