diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f471b38251..1e7d360be3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -364,7 +364,7 @@ def get_checksum_for(self, checksums, filename=None, index=None): else: raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) - def fetch_source(self, source, checksum=None, extension=False): + def fetch_source(self, source, checksum=None, extension=False, download_instructions=None): """ Get a specific source (tarball, iso, url) Will be tested for existence or can be located @@ -400,7 +400,8 @@ def fetch_source(self, source, checksum=None, extension=False): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] path = self.obtain_file(filename, extension=extension, download_filename=download_filename, - force_download=force_download, urls=source_urls, git_config=git_config) + force_download=force_download, urls=source_urls, git_config=git_config, + download_instructions=download_instructions) if path is None: raise EasyBuildError('No file found for source %s', filename) @@ -586,7 +587,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source['source_urls'] = source_urls if fetch_files: - src = self.fetch_source(source, checksums, extension=True) + src = self.fetch_source(source, checksums, extension=True, + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -682,7 +684,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None): + git_config=None, download_instructions=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -869,8 +871,19 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + error_msg = "Couldn't find file %s anywhere, " + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + if download_instructions is not None and download_instructions != "": + msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + print_msg(msg, prefix=False, stderr=True) + error_msg += "please follow the download instructions above, and make the file available " + error_msg += "in the active source path (%s)" % ':'.join(source_paths()) + else: + error_msg += "and downloading it didn't work either... " + error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths) + + raise EasyBuildError(error_msg, filename) # # GETTER/SETTER UTILITY FUNCTIONS diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index eeed425420..a1f1fef043 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -90,6 +90,7 @@ 'checksums': [[], "Checksums for sources and patches", BUILD], 'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD], 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD], + 'download_instructions': ['', "Specify steps to aquire necessary file, if obtaining it is difficult", BUILD], 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6da8ec6815..960af782bf 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1480,6 +1480,110 @@ def test_fetch_sources(self): error_pattern = "Found one or more unexpected keys in 'sources' specification: {'nosuchkey': 'foobar'}" self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_sources, sources, checksums=[]) + def test_download_instructions(self): + """Test use of download_instructions easyconfig parameter.""" + orig_test_ec = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'software_with_missing_sources'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "sources = [SOURCE_TAR_GZ]", + "exts_list = [", + " ('ext_with_missing_sources', '0.0', {", + " 'sources': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.contents = orig_test_ec + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + common_error_pattern = "^Couldn't find file software_with_missing_sources-0.0.tar.gz anywhere" + error_pattern = common_error_pattern + ", and downloading it didn't work either" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + download_instructions = "download_instructions = 'Manual download from example.com required'" + sources = "sources = [SOURCE_TAR_GZ]" + self.contents = self.contents.replace(sources, download_instructions + '\n' + sources) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = common_error_pattern + ", please follow the download instructions above" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # create dummy source file + write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') + + # now downloading of sources for extension should fail + # top-level download instructions are printed (because there's nothing else) + error_pattern = "^Couldn't find file ext_with_missing_sources-0.0.tar.gz anywhere" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # wipe top-level download instructions, try again + self.contents = self.contents.replace(download_instructions, '') + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + # no download instructions printed anymore now + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stdout, '') + + # inject download instructions for extension + download_instructions = ' ' * 8 + "'download_instructions': " + download_instructions += "'Extension sources must be downloaded via example.com'," + sources = "'sources': [SOURCE_TAR_GZ]," + self.contents = self.contents.replace(sources, sources + '\n' + download_instructions) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") + self.assertEqual(stdout, '') + + # create dummy source file for extension + write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') + + # no more errors, all source files found (so no download instructions printed either) + self.mock_stderr(True) + self.mock_stdout(True) + eb.fetch_step() + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + self.assertEqual(stdout, '') + def test_fetch_patches(self): """Test fetch_patches method.""" testdir = os.path.abspath(os.path.dirname(__file__))