From dcc881f95ac569137d9899e0afaa5c6e1135c81a Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:10:26 +0100 Subject: [PATCH 01/14] EHSTPCR-1071: correct tapplus and data distribution --- astroquery/esa/hubble/__init__.py | 27 +- astroquery/esa/hubble/core.py | 247 +++++++++++------- astroquery/esa/hubble/tests/__init__.py | 7 +- .../esa/hubble/tests/dummy_tap_handler.py | 16 +- astroquery/esa/hubble/tests/setup_package.py | 8 + .../esa/hubble/tests/test_esa_hubble.py | 110 ++++---- .../hubble/tests/test_esa_hubble_remote.py | 35 +-- docs/esa/hubble/hubble.rst | 40 +-- 8 files changed, 273 insertions(+), 217 deletions(-) diff --git a/astroquery/esa/hubble/__init__.py b/astroquery/esa/hubble/__init__.py index b8178df66b..427d0e0154 100644 --- a/astroquery/esa/hubble/__init__.py +++ b/astroquery/esa/hubble/__init__.py @@ -1,35 +1,30 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst """ - -@author: Javier Duran -@contact: javier.duran@sciops.esa.int +========== +eHST Init +========== European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 13 Aug. 2018 - """ from astropy import config as _config +from astropy.config import paths +import os class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.esa.hubble`. """ - DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/" - "ehst-sl-server/servlet/data-action", - "Main url for retriving hst files") - METADATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/" - "ehst-sl-server/servlet/" - "metadata-action", - "Main url for retriving hst metadata") - TARGET_ACTION = _config.ConfigItem("http://archives.esac.esa.int/" - "ehst-sl-server/servlet/" - "targetresolver-action", - "Main url for solving targets") + EHST_TAP_SERVER = _config.ConfigItem("https://hst.esac.esa.int/tap-server/tap", "eHST TAP Server") + EHST_TARGET_ACTION = _config.ConfigItem("servlet/target-resolver?", "eHST Target Resolver") + EHST_MESSAGES = _config.ConfigItem("notification?action=GetNotifications", "eHST Messages") TIMEOUT = 60 + cache_location = os.path.join(paths.get_cache_dir(), 'astroquery/ehst', ) + conf = Conf() diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index 9e879e3906..d4e976aa7a 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -1,27 +1,29 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - ====================== eHST Astroquery Module ====================== - -@author: Javier Duran -@contact: javier.duran@sciops.esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 13 Aug. 2018 - - """ + from astropy import units from astropy.coordinates import SkyCoord from astropy.coordinates import Angle -from astroquery.utils.tap.core import TapPlus + +from requests.exceptions import ConnectionError + +from astroquery.exceptions import RemoteServiceError +from astroquery.utils.tap import TapPlus +from astroquery.utils import commons from astroquery.query import BaseQuery import shutil +import json +from urllib.parse import urlencode +import warnings +from astropy.utils.exceptions import AstropyDeprecationWarning from . import conf from astroquery import log @@ -33,24 +35,20 @@ class ESAHubbleClass(BaseQuery): """ Class to init ESA Hubble Module and communicate with eHST TAP """ - - data_url = conf.DATA_ACTION - metadata_url = conf.METADATA_ACTION - target_url = conf.TARGET_ACTION TIMEOUT = conf.TIMEOUT calibration_levels = {"AUXILIARY": 0, "RAW": 1, "CALIBRATED": 2, "PRODUCT": 3} - product_types = ["PRODUCT", "SCIENCE_PRODUCT", "POSTCARD"] + product_types = ["SCIENCE", "PREVIEW", "THUMBNAIL" or "AUXILIARY"] copying_string = "Copying file to {0}..." - def __init__(self, tap_handler=None): - super().__init__() - + def __init__(self, *, tap_handler=None, show_messages=True): if tap_handler is None: - self._tap = TapPlus(url="https://hst.esac.esa.int" - "/tap-server/tap") + self._tap = TapPlus(url=conf.EHST_TAP_SERVER, + data_context='data', client_id="ASTROQUERY") else: self._tap = tap_handler + if show_messages: + self.get_status_messages() def download_product(self, observation_id, *, calibration_level=None, filename=None, verbose=False, product_type=None): @@ -78,39 +76,43 @@ def download_product(self, observation_id, *, calibration_level=None, flag to display information about the process product_type : string type of product retrieval, optional - PRODUCT, SCIENCE_PRODUCT or POSTCARD + SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY Returns ------- None. It downloads the observation indicated """ - params = {"OBSERVATION_ID": observation_id, - "USERNAME": "ehst-astroquery"} - url = self.data_url + "?OBSERVATION_ID=" + observation_id - url += "&USERNAME=" + "ehst-astroquery" + params = {"OBSERVATIONID": observation_id, + "TAPCLIENT": "ASTROQUERY", + "RETRIEVAL_TYPE": "OBSERVATION"} if filename is None: filename = observation_id + ".tar" if calibration_level: - params["CALIBRATION_LEVEL"] = calibration_level - url += "&CALIBRATION_LEVEL=" + calibration_level + params["CALIBRATIONLEVEL"] = calibration_level + # Product type check to ensure backwards compatibility + product_type = self.__set_product_type(product_type) if product_type: self.__validate_product_type(product_type) - params["RETRIEVAL_TYPE"] = product_type - filename = self._get_product_filename(product_type, filename) - url += "&RETRIEVAL_TYPE=" + params["RETRIEVAL_TYPE"] + params["PRODUCTTYPE"] = product_type - response = self._request('GET', self.data_url, save=True, cache=True, - params=params) + filename = self._get_product_filename(product_type, filename) + response = self._tap.load_data(params, filename, verbose=verbose) - if verbose: - log.info(url) - log.info(self.copying_string.format(filename)) + return filename - shutil.move(response, filename) + def __set_product_type(self, product_type): + if product_type: + if 'SCIENCE_PRODUCT' in product_type: + return 'SCIENCE' + elif 'PRODUCT' in product_type: + return None + elif 'POSTCARD' in product_type: + return 'PREVIEW' + return product_type def get_member_observations(self, observation_id): """ @@ -159,7 +161,7 @@ def get_hap_hst_link(self, observation_id): oids = self._select_related_members(observation_id) elif 'HST' in observation_type: query = f"select observation_id from ehst.observation where obs_type='HAP Simple' and members like '%{observation_id}%'" - job = self.query_hst_tap(query=query) + job = self.query_tap(query=query) oids = job["observation_id"].pformat(show_name=False) else: raise ValueError("Invalid observation id") @@ -184,7 +186,7 @@ def get_observation_type(self, observation_id): raise ValueError("Please input an observation id") query = f"select obs_type from ehst.observation where observation_id='{observation_id}'" - job = self.query_hst_tap(query=query) + job = self.query_tap(query=query) if any(job["obs_type"]): obs_type = self._get_decoded_string(string=job["obs_type"][0]) else: @@ -193,13 +195,13 @@ def get_observation_type(self, observation_id): def _select_related_members(self, observation_id): query = f"select members from ehst.observation where observation_id='{observation_id}'" - job = self.query_hst_tap(query=query) + job = self.query_tap(query=query) oids = self._get_decoded_string(string=job["members"][0]).replace("caom:HST/", "").split(" ") return oids def _select_related_composite(self, observation_id): query = f"select observation_id from ehst.observation where members like '%{observation_id}%'" - job = self.query_hst_tap(query=query) + job = self.query_tap(query=query) oids = job["observation_id"].pformat(show_name=False) return oids @@ -210,15 +212,17 @@ def __validate_product_type(self, product_type): def _get_product_filename(self, product_type, filename): if (product_type == "PRODUCT"): return filename - elif (product_type == "SCIENCE_PRODUCT"): + elif (product_type == "SCIENCE"): log.info("This is a SCIENCE_PRODUCT, the filename will be " f"renamed to {filename}.fits.gz") return f"{filename}.fits.gz" - else: + elif (product_type == "THUMBNAIL" or product_type == "PREVIEW"): log.info("This is a POSTCARD, the filename will be " f"renamed to {filename}.jpg") return f"{filename}.jpg" + return filename + def get_artifact(self, artifact_id, filename=None, verbose=False): """ Download artifacts from EHST. Artifact is a single Hubble product file. @@ -240,20 +244,15 @@ def get_artifact(self, artifact_id, filename=None, verbose=False): None. It downloads the artifact indicated """ - params = {"ARTIFACT_ID": artifact_id, "USERNAME": "ehst-astroquery"} - response = self._request('GET', self.data_url, save=True, cache=True, - params=params) + params = {"RETRIEVAL_TYPE": "PRODUCT", "ARTIFACTID": artifact_id, "TAPCLIENT": "ASTROQUERY"} if filename is None: filename = artifact_id - if verbose: - log.info(self.data_url + "?ARTIFACT_ID=" + artifact_id + - "&USERNAME=ehst-astroquery") - log.info(self.copying_string.format(filename)) + self._tap.load_data(params, filename, verbose=verbose) - shutil.move(response, filename) + return filename - def get_postcard(self, observation_id, calibration_level="RAW", + def get_postcard(self, observation_id, *, calibration_level="RAW", resolution=256, filename=None, verbose=False): """ Download postcards from EHST @@ -284,28 +283,30 @@ def get_postcard(self, observation_id, calibration_level="RAW", None. It downloads the observation postcard indicated """ - params = {"RETRIEVAL_TYPE": "POSTCARD", - "OBSERVATION_ID": observation_id, - "CALIBRATION_LEVEL": calibration_level, - "RESOLUTION": resolution, - "USERNAME": "ehst-astroquery"} + params = {"RETRIEVAL_TYPE": "OBSERVATION", + "OBSERVATIONID": observation_id, + "PRODUCTTYPE": "PREVIEW", + "TAPCLIENT": "ASTROQUERY"} + if calibration_level: + params["CALIBRATIONLEVEL"] = calibration_level - response = self._request('GET', self.data_url, save=True, cache=True, - params=params) + if resolution: + params["PRODUCTTYPE"] = self.__get_product_type_by_resolution(resolution) if filename is None: filename = observation_id - if verbose: - log.info(self.data_url + - "&".join(["?RETRIEVAL_TYPE=POSTCARD", - "OBSERVATION_ID=" + observation_id, - "CALIBRATION_LEVEL=" + calibration_level, - "RESOLUTION=" + str(resolution), - "USERNAME=ehst-astroquery"])) - log.info(self.copying_string.format(filename)) + self._tap.load_data(params, filename, verbose=verbose) - shutil.move(response, filename) + return filename + + def __get_product_type_by_resolution(self, resolution): + if resolution == 256: + return 'THUMBNAIL' + elif resolution == 1024: + return 'PREVIEW' + else: + raise ValueError("Resolution must be 256 or 1024") def cone_search(self, coordinates, radius, filename=None, output_format='votable', cache=True, @@ -370,10 +371,10 @@ def cone_search(self, coordinates, radius, filename=None, if verbose: log.info(query) - table = self.query_hst_tap(query=query, async_job=async_job, - output_file=filename, - output_format=output_format, - verbose=verbose) + table = self.query_tap(query=query, async_job=async_job, + output_file=filename, + output_format=output_format, + verbose=verbose) return table def cone_search_criteria(self, radius, target=None, @@ -482,7 +483,7 @@ def cone_search_criteria(self, radius, target=None, if verbose: log.info(query) - table = self.query_hst_tap(query=query, async_job=async_job, + table = self.query_tap(query=query, async_job=async_job, output_file=filename, output_format=output_format, verbose=verbose) @@ -491,17 +492,21 @@ def cone_search_criteria(self, radius, target=None, def _query_tap_target(self, target): try: params = {"TARGET_NAME": target, - "RESOLVER_TYPE": "SN", - "FORMAT": "json"} - target_response = self._request('GET', - self.target_url, - cache=True, - params=params) - target_result = target_response.json()['data'][0] - ra = target_result['RA_DEGREES'] - dec = target_result['DEC_DEGREES'] + "RESOLVER_TYPE": "ALL", + "FORMAT": "json", + "TAPCLIENT": "ASTROQUERY"} + + subContext = conf.EHST_TARGET_ACTION + connHandler = self._tap._TapPlus__getconnhandler() + data = connHandler.url_encode(params) + target_response = connHandler.execute_secure(subContext, data, True) + for line in target_response: + target_result = json.loads(line.decode("utf-8")) + if target_result['objects']: + ra = target_result['objects'][0]['raDegrees'] + dec = target_result['objects'][0]['decDegrees'] return SkyCoord(ra=ra, dec=dec, unit="deg") - except KeyError: + except (ValueError, KeyError): raise ValueError("This target cannot be resolved") def query_metadata(self, output_format='votable', verbose=False): @@ -541,6 +546,43 @@ def query_target(self, name, *, filename=None, output_format='votable', return table + def query_tap(self, query, async_job=False, output_file=None, + output_format="votable", verbose=False): + """Launches a synchronous or asynchronous job to query the HST tap + + Parameters + ---------- + query : str, mandatory + query (adql) to be executed + async_job : bool, optional, default 'False' + executes the query (job) in asynchronous/synchronous mode (default + synchronous) + output_file : str, optional, default None + file name where the results are saved if dumpToFile is True. + If this parameter is not provided, the jobid is used instead + output_format : str, optional, default 'votable' + results format + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + A table object + """ + if async_job: + job = self._tap.launch_job_async(query=query, + output_file=output_file, + output_format=output_format, + verbose=verbose, + dump_to_file=output_file is not None) + else: + job = self._tap.launch_job(query=query, output_file=output_file, + output_format=output_format, + verbose=verbose, + dump_to_file=output_file is not None) + table = job.get_results() + return table + def query_hst_tap(self, query, async_job=False, output_file=None, output_format="votable", verbose=False): """Launches a synchronous or asynchronous job to query the HST tap @@ -564,19 +606,12 @@ def query_hst_tap(self, query, async_job=False, output_file=None, ------- A table object """ - if async_job: - job = self._tap.launch_job_async(query=query, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=output_file is not None) - else: - job = self._tap.launch_job(query=query, output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=output_file is not None) - table = job.get_results() - return table + warnings.warn( + "Use of query_hst_tap method is no longer supported. Please use" + " query_tap method instead, with the same arguments.", + AstropyDeprecationWarning) + self.query_tap(query=query, async_job=False, output_file=None, + output_format="votable", verbose=False) def query_criteria(self, calibration_level=None, data_product_type=None, intent=None, @@ -661,10 +696,10 @@ def query_criteria(self, calibration_level=None, log.info(query) if get_query: return query - table = self.query_hst_tap(query=query, async_job=async_job, - output_file=output_file, - output_format=output_format, - verbose=verbose) + table = self.query_tap(query=query, async_job=async_job, + output_file=output_file, + output_format=output_format, + verbose=verbose) return table def __get_calibration_level(self, calibration_level): @@ -717,6 +752,22 @@ def get_tables(self, only_names=True, verbose=False): else: return tables + def get_status_messages(self): + """Retrieve the messages to inform users about + the status of eHST TAP + """ + + try: + subContext = conf.EHST_MESSAGES + connHandler = self._tap._TapPlus__getconnhandler() + response = connHandler.execute_tapget(subContext, False) + if response.status == 200: + for line in response: + string_message = line.decode("utf-8") + print(string_message[string_message.index('=')+1:]) + except OSError: + print("Status messages could not be retrieved") + def get_columns(self, table_name, only_names=True, verbose=False): """Get the available columns for a table in EHST TAP service diff --git a/astroquery/esa/hubble/tests/__init__.py b/astroquery/esa/hubble/tests/__init__.py index 110f6e6b41..e1a2346afc 100644 --- a/astroquery/esa/hubble/tests/__init__.py +++ b/astroquery/esa/hubble/tests/__init__.py @@ -1,11 +1,10 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - -@author: Javier Duran -@contact: javier.duran@sciops.esa.int +========== +eHST Init +========== European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 13 Aug. 2018 """ diff --git a/astroquery/esa/hubble/tests/dummy_tap_handler.py b/astroquery/esa/hubble/tests/dummy_tap_handler.py index 3f5c65c6a5..4d38d7d716 100644 --- a/astroquery/esa/hubble/tests/dummy_tap_handler.py +++ b/astroquery/esa/hubble/tests/dummy_tap_handler.py @@ -1,16 +1,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - -@author: Javier Duran -@contact: javier.duran@sciops.esa.int +====================== +eHST Dummy Tap Handler +====================== European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 30 Aug. 2018 - - """ + from astroquery.utils.tap.model.taptable import TapTableMeta from astroquery.utils.tap.model.job import Job @@ -71,3 +69,9 @@ def load_tables(self, table = TapTableMeta() table.name = "table" return [table] + + def load_data(self, params_dict, output_file=None, verbose=False): + self.__invokedMethod = 'load_data' + self._parameters['params_dict'] = params_dict + self._parameters['output_file'] = output_file + self._parameters['verbose'] = verbose diff --git a/astroquery/esa/hubble/tests/setup_package.py b/astroquery/esa/hubble/tests/setup_package.py index 85aba30125..14527639ac 100644 --- a/astroquery/esa/hubble/tests/setup_package.py +++ b/astroquery/esa/hubble/tests/setup_package.py @@ -1,5 +1,13 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +======================= +eHST Test Setup Package +======================= +European Space Astronomy Centre (ESAC) +European Space Agency (ESA) + +""" import os diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index d3613557c4..a5d8ea1245 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -1,16 +1,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - -@author: Javier Duran -@contact: javier.duran@sciops.esa.int +============== +eHST Tap Tests +============== European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 13 Aug. 2018 - - """ + import os import shutil from pathlib import Path @@ -86,11 +84,11 @@ def get_dummy_tap_handler(self): return dummyTapHandler def test_download_product_errors(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) with pytest.raises(ValueError) as err: ehst.download_product(observation_id="J6FL25S4Q", - product_type="SCIENCE") + product_type="DUMMY") assert "This product_type is not allowed" in err.value.args[0] def test_download_product_by_calibration(self, tmp_path): @@ -98,7 +96,7 @@ def test_download_product_by_calibration(self, tmp_path): 'cal_level': "RAW", 'filename': Path(tmp_path, "J6FL25S4Q.vot.test"), 'verbose': True} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], calibration_level=parameters['cal_level'], filename=parameters['filename'], @@ -109,26 +107,26 @@ def test_download_product_by_product_type(self, tmp_path): 'product_type': "SCIENCE_PRODUCT", 'filename': Path(tmp_path, "J6FL25S4Q.vot.test"), 'verbose': True} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], product_type=parameters['product_type'], filename=parameters['filename'], verbose=parameters['verbose']) parameters['product_type'] = "PRODUCT" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], product_type=parameters['product_type'], filename=parameters['filename'], verbose=parameters['verbose']) parameters['product_type'] = "POSTCARD" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], product_type=parameters['product_type'], filename=parameters['filename'], verbose=parameters['verbose']) def test_get_postcard(self, tmp_path): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.get_postcard(observation_id="X0MC5101T", filename=Path(tmp_path, "X0MC5101T.vot"), verbose=True) @@ -138,14 +136,14 @@ def test_get_postcard(self, tmp_path): def test_query_target(self, mock_query_tap_target, mock_cone_search): mock_query_tap_target.return_value = 10, 10 mock_cone_search.return_value = "test" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) table = ehst.query_target(name="test") assert table == "test" def test_cone_search(self): coords = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) parameters = {'coordinates': coords, 'radius': 0.0, 'filename': 'file_cone', @@ -181,7 +179,7 @@ def test_cone_search_coords(self): 'cache': True, 'verbose': True} - ehst = ESAHubbleClass(dummyTapHandler) + ehst = ESAHubbleClass(tap_handler=dummyTapHandler, show_messages=False) ehst.cone_search(parameters['coordinates'], parameters['radius'], parameters['filename'], @@ -194,7 +192,7 @@ def test_cone_search_coords(self): assert "Coordinates must be either a string or " \ "astropy.coordinates" in err.value.args[0] - def test_query_hst_tap(self): + def test_query_tap(self): parameters = {'query': "select top 10 * from hsc_v2.hubble_sc2", 'async_job': False, 'output_file': "test2.vot", @@ -205,8 +203,8 @@ def test_query_hst_tap(self): 'output_format': "votable", 'verbose': False} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) - ehst.query_hst_tap(parameters['query'], parameters['async_job'], + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst.query_tap(parameters['query'], parameters['async_job'], parameters['output_file'], parameters['output_format'], parameters['verbose']) self.get_dummy_tap_handler().check_call("launch_job", parameters2) @@ -216,11 +214,11 @@ def test_get_tables(self): 'verbose': True} DummyHubbleTapHandler("get_tables", parameters) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.get_tables(True, True) def test_get_artifact(self, tmp_path): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) path = Path(tmp_path, "w0ji0v01t_c2f.fits.gz") ehst.get_artifact(path) @@ -230,7 +228,7 @@ def test_get_columns(self): 'verbose': True} dummyTapHandler = DummyHubbleTapHandler("get_columns", parameters) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.get_columns("table", True, True) dummyTapHandler.check_call("get_columns", parameters) @@ -246,7 +244,7 @@ def test_query_criteria(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) test_query = ehst.query_criteria(parameters1['calibration_level'], parameters1['data_product_type'], parameters1['intent'], @@ -287,7 +285,7 @@ def test_query_criteria_numeric_calibration(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) test_query = ehst.query_criteria(parameters1['calibration_level'], parameters1['data_product_type'], parameters1['intent'], @@ -355,7 +353,7 @@ def test_cone_search_criteria(self): 'output_file': "output_test_query_by_criteria.vot.gz", 'output_format': "votable", 'verbose': False} - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) query_criteria_query = "select o.*, p.calibration_level, " \ "p.data_product_type, pos.ra, pos.dec from " \ "ehst.observation AS o JOIN ehst.plane as p " \ @@ -415,7 +413,7 @@ def test_cone_search_criteria(self): "parameter." in err.value.args[0] def test_query_criteria_no_params(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.query_criteria(async_job=False, output_file="output_test_query_" "by_criteria.vot.gz", @@ -432,7 +430,7 @@ def test_query_criteria_no_params(self): dummy_tap_handler.check_call("launch_job", parameters) def test_empty_list(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) with pytest.raises(ValueError) as err: ehst.query_criteria(instrument_name=[1], async_job=False, @@ -444,54 +442,54 @@ def test_empty_list(self): "elements that are not strings" in err.value.args[0] def test__get_decoded_string(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy = '\x74\x65\x73\x74' decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == 'test' def test__get_decoded_string_unicodedecodeerror(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy = '\xd0\x91' decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == dummy def test__get_decoded_string_attributeerror(self): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy = True decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == dummy - @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'query_tap') def test__select_related_composite(self, mock_query): arr = {'a': np.array([1, 4], dtype=np.int32), 'b': [2.0, 5.0], 'observation_id': ['x', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst._select_related_composite(dummy_obs_id) assert oids == ['x', 'y'] - @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'query_tap') def test__select_related_members(self, mock_query): arr = {'a': np.array([1, 4], dtype=np.int32), 'b': [2.0, 5.0], 'members': ['caom:HST/test', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst._select_related_members(dummy_obs_id) assert oids == ['test'] - @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'query_tap') def test_get_observation_type(self, mock_query): arr = {'a': np.array([1, 4], dtype=np.int32), 'b': [2.0, 5.0], 'obs_type': ['HST Test', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst.get_observation_type(dummy_obs_id) @@ -499,23 +497,23 @@ def test_get_observation_type(self, mock_query): def test_get_observation_type_obs_id_none_valueerror(self): with pytest.raises(ValueError): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = None ehst.get_observation_type(dummy_obs_id) - @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'query_tap') def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query): with pytest.raises(ValueError): arr = {'a': np.array([], dtype=np.int32), 'b': [], 'obs_type': []} data_table = Table(arr) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = '1234' ehst.get_observation_type(dummy_obs_id) - @patch.object(ESAHubbleClass, 'query_hst_tap') + @patch.object(ESAHubbleClass, 'query_tap') @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_hst_link(self, mock_observation_type, mock_query): mock_observation_type.return_value = "HST" @@ -523,7 +521,7 @@ def test_get_hst_link(self, mock_observation_type, mock_query): 'b': [2.0], 'observation_id': ['1234']} data_table = Table(arr) - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst.get_hap_hst_link(dummy_obs_id) @@ -534,7 +532,7 @@ def test_get_hst_link(self, mock_observation_type, mock_query): def test_get_hap_link(self, mock_select_related_members, mock_observation_type): mock_select_related_members.return_value = 'test' mock_observation_type.return_value = "HAP" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" oids = ehst.get_hap_hst_link(dummy_obs_id) assert oids == 'test' @@ -543,7 +541,7 @@ def test_get_hap_link(self, mock_select_related_members, mock_observation_type): def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "valueerror" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" ehst.get_hap_hst_link(dummy_obs_id) @@ -551,7 +549,7 @@ def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): def test_get_hap_hst_link_compositeerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "HAP Composite" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" ehst.get_hap_hst_link(dummy_obs_id) @@ -559,7 +557,7 @@ def test_get_hap_hst_link_compositeerror(self, mock_observation_type): @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_member_observations_composite(self, mock_observation_type, mock_select_related_members): mock_observation_type.return_value = "Composite" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_select_related_members.return_value = 'test' dummy_obs_id = "1234" oids = ehst.get_member_observations(dummy_obs_id) @@ -569,7 +567,7 @@ def test_get_member_observations_composite(self, mock_observation_type, mock_sel @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_member_observations_simple(self, mock_observation_type, mock_select_related_composite): mock_observation_type.return_value = "Simple" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_select_related_composite.return_value = 'test' dummy_obs_id = "1234" oids = ehst.get_member_observations(dummy_obs_id) @@ -579,27 +577,27 @@ def test_get_member_observations_simple(self, mock_observation_type, mock_select def test_get_member_observations_invalid_id_valueerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "valueerror" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" ehst.get_member_observations(dummy_obs_id) @patch.object(ESAHubbleClass, 'query_criteria') @patch.object(ESAHubbleClass, '_query_tap_target') - @patch.object(ESAHubbleClass, 'query_hst_tap') - def test_cone_search_criteria_only_target(self, mock_query_hst_tap, mock__query_tap_target, mock_query_criteria): + @patch.object(ESAHubbleClass, 'query_tap') + def test_cone_search_criteria_only_target(self, mock_query_tap, mock__query_tap_target, mock_query_criteria): mock_query_criteria.return_value = "Simple query" mock__query_tap_target.return_value = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - mock_query_hst_tap.return_value = "table" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query_tap.return_value = "table" + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) oids = ehst.cone_search_criteria(target="m11", radius=1) assert oids == 'table' @patch.object(ESAHubbleClass, 'query_criteria') - @patch.object(ESAHubbleClass, 'query_hst_tap') - def test_cone_search_criteria_only_coordinates(self, mock_query_hst_tap, mock_query_criteria): + @patch.object(ESAHubbleClass, 'query_tap') + def test_cone_search_criteria_only_coordinates(self, mock_query_tap, mock_query_criteria): mock_query_criteria.return_value = "Simple query" - mock_query_hst_tap.return_value = "table" - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + mock_query_tap.return_value = "table" + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) oids = ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", radius=1) assert oids == 'table' @@ -607,5 +605,5 @@ def test_cone_search_criteria_only_coordinates(self, mock_query_hst_tap, mock_qu def test_cone_search_criteria_typeerror(self, mock_query_criteria): mock_query_criteria.return_value = "Simple query" with pytest.raises(TypeError): - ehst = ESAHubbleClass(self.get_dummy_tap_handler()) + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", target="m11", radius=1) diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index e69a71f416..3da24bbbfd 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -1,16 +1,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - -@author: Javier Espinosa -@contact: javier.espinosa@sciops.esa.int +================= +eHST Remote Tests +================= European Space Astronomy Centre (ESAC) European Space Agency (ESA) -Created on 13 Jan. 2021 - - """ + import tempfile import os @@ -19,6 +17,7 @@ from astroquery.esa.hubble import ESAHubble from astropy import coordinates import random +from astropy.table import Table esa_hubble = ESAHubble() @@ -40,26 +39,26 @@ def remove_last_job(): @pytest.mark.remote_data class TestEsaHubbleRemoteData: - obs_query = "select top 2050 o.observation_id from ehst.observation o" + obs_query = "select top 2050 a.observation_id from ehst.archive a" - top_obs_query = "select top 100 o.observation_id from ehst.observation o" + top_obs_query = "select top 100 a.observation_id from ehst.archive a" - hst_query = "select top 50 o.observation_id from ehst.observation " \ - "o where o.collection='HST'" + hst_query = "select top 50 a.observation_id from ehst.archive " \ + "a where a.collection='HST'" top_artifact_query = "select top 50 a.artifact_id, a.observation_id " \ " from ehst.artifact a" temp_folder = create_temp_folder() - def test_query_hst_tap_async(self): - result = esa_hubble.query_hst_tap(self.top_obs_query, async_job=True) + def test_query_tap_async(self): + result = esa_hubble.query_tap(self.top_obs_query, async_job=True) assert len(result) > 10 assert "observation_id" in result.keys() remove_last_job() def test_download_product(self): - result = esa_hubble.query_hst_tap(self.hst_query) + result = esa_hubble.query_tap(self.hst_query) observation_id = random.choice(result['observation_id']) temp_file = self.temp_folder.name + "/" + observation_id + ".tar" esa_hubble.download_product(observation_id=observation_id, @@ -67,7 +66,7 @@ def test_download_product(self): assert os.path.exists(temp_file) def test_get_artifact(self): - result = esa_hubble.query_hst_tap(self.top_artifact_query) + result = esa_hubble.query_tap(self.top_artifact_query) assert "artifact_id" in result.keys() artifact_id = random.choice(result["artifact_id"]) temp_file = self.temp_folder.name + "/" + artifact_id + ".gz" @@ -78,7 +77,9 @@ def test_cone_search(self): esa_hubble = ESAHubble() c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') temp_file = self.temp_folder.name + "/cone_search_m31_5.vot" - table = esa_hubble.cone_search(c, 7, temp_file, verbose=True) + compressed_temp_file = self.temp_folder.name + "/cone_search_m31_5.vot.gz" + # open & extracting the file + table = esa_hubble.cone_search(c, 7, compressed_temp_file, verbose=True) assert 'observation_id' in table.columns assert len(table) > 0 remove_last_job() @@ -115,6 +116,6 @@ def test_hst_simple_to_hap_simple(self): assert result == ['hst_16316_71_acs_sbc_f150lp_jec071i9'] def test_query_target(self): - temp_file = self.temp_folder.name + "/" + "m31_query.xml" - table = esa_hubble.query_target(name="m3", filename=temp_file) + compressed_temp_file = self.temp_folder.name + "/" + "m31_query.xml.gz" + table = esa_hubble.query_target(name="m3", filename=compressed_temp_file) assert 'observation_id' in table.columns diff --git a/docs/esa/hubble/hubble.rst b/docs/esa/hubble/hubble.rst index 4c51e22d84..76d9172774 100644 --- a/docs/esa/hubble/hubble.rst +++ b/docs/esa/hubble/hubble.rst @@ -31,35 +31,35 @@ This function allows the user to download products based on their observation ID a required calibration_level (RAW, CALIBRATED, PRODUCT or AUXILIARY) and/or product type (PRODUCT, SCIENCE_PRODUCT or POSTCARD). -This will download all files for the raw calibration level of the observation 'J6FL25S4Q' and it will store them in a tar called -'raw_data_for_J6FL25S4Q.tar'. +This will download all files for the raw calibration level of the observation 'j6fl25s4q' and it will store them in a tar called +'raw_data_for_j6fl25s4q.tar'. .. doctest-remote-data:: >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.download_product(observation_id="J6FL25S4Q", calibration_level="RAW", - ... filename="raw_data_for_J6FL25S4Q.tar") # doctest: +IGNORE_OUTPUT + >>> esahubble.download_product(observation_id="j6fl25s4q", calibration_level="RAW", + ... filename="raw_data_for_j6fl25s4q.fits") # doctest: +IGNORE_OUTPUT -This will download the science files associated to the observation 'J6FL25S4Q' and it will store them in a file called -'science_data_for_J6FL25S4Q.tar.fits.gz', modifying the filename provided to ensure that the extension of the file is correct. +This will download the science files associated to the observation 'j6fl25s4q' and it will store them in a file called +'science_data_for_j6fl25s4q.tar.fits.gz', modifying the filename provided to ensure that the extension of the file is correct. .. doctest-remote-data:: >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.download_product(observation_id="J6FL25S4Q", product_type="SCIENCE_PRODUCT", - ... filename="science_data_for_J6FL25S4Q.tar") # doctest: +IGNORE_OUTPUT + >>> esahubble.download_product(observation_id="j6fl25s4q", product_type="SCIENCE_PRODUCT", + ... filename="science_data_for_j6fl25s4q.fits") # doctest: +IGNORE_OUTPUT -This third case will download the science files associated to the observation 'J6FL25S4Q' in raw calibration level and it will store them in a file called -'science_raw_data_for_J6FL25S4Q.fits.gz', modifying the filename provided to ensure that the extension of the file is correct. +This third case will download the science files associated to the observation 'j6fl25s4q' in raw calibration level and it will store them in a file called +'science_raw_data_for_j6fl25s4q.fits.gz', modifying the filename provided to ensure that the extension of the file is correct. .. doctest-remote-data:: >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.download_product(observation_id="J6FL25S4Q", calibration_level="RAW", - ... filename="science_raw_data_for_J6FL25S4Q", product_type="SCIENCE_PRODUCT") # doctest: +IGNORE_OUTPUT + >>> esahubble.download_product(observation_id="j6fl25s4q", calibration_level="RAW", + ... filename="science_raw_data_for_j6fl25s4q", product_type="SCIENCE_PRODUCT") # doctest: +IGNORE_OUTPUT --------------------------- 2. Getting Hubble postcards @@ -69,11 +69,11 @@ This third case will download the science files associated to the observation 'J >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.get_postcard("J6FL25S4Q", "RAW", 256, "raw_postcard_for_J6FL25S4Q.jpg") # doctest: +IGNORE_OUTPUT + >>> esahubble.get_postcard(observation_id="j6fl25s4q", calibration_level="RAW", resolution=256, filename="raw_postcard_for_j6fl25s4q.jpg") # doctest: +IGNORE_OUTPUT This will download the postcard for the observation 'J8VP03010' with low resolution (256) and it will stored in a jpg called -'raw_postcard_for_J6FL25S4Q.jpg'. Resolution of 1024 is also available. +'raw_postcard_for_j6fl25s4q.jpg'. Resolution of 1024 is also available. Calibration levels can be RAW, CALIBRATED, PRODUCT or AUXILIARY. @@ -88,7 +88,7 @@ Note: Artifact is a single Hubble product file. >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.get_artifact("w0ji0v01t_c2f.fits.gz") + >>> esahubble.get_artifact("w0ji0v01t_c2f.fits") This will download the compressed artifact 'w0ji0v01t_c2f.fits.gz'. 'w0ji0v01t_c2f.fits' is the name of the Hubble @@ -104,12 +104,12 @@ The query_target function queries the name of the target as given by the propose >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> table = esahubble.query_target("m31", filename="m31_query.xml") # doctest: +IGNORE_OUTPUT + >>> table = esahubble.query_target("m31", filename="m31_query.xml.gz") # doctest: +IGNORE_OUTPUT This will retrieve a table with the output of the query. It will also download a file storing all metadata for all observations associated with target name 'm31'. The result of the query will be stored in -file 'm31_query.xml'. +file 'm31_query.xml.gz'. ----------------------------------------------------------------- 5. Querying observations by search criteria in the Hubble archive @@ -319,11 +319,11 @@ This last example will provide the ADQL query based on the criteria defined by t >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() >>> c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - >>> table = esahubble.cone_search(c, 7, "cone_search_m31_5.vot") + >>> table = esahubble.cone_search(c, 7, "cone_search_m31_5.vot.gz") This will perform a cone search with radius 7 arcmins. The result of the query will be returned and stored in the votable file -'cone_search_m31_5.vot'. If no filename is defined and the "save" tag is True, +'cone_search_m31_5.vot.gz'. If no filename is defined and the "save" tag is True, the module will provide a default name. It is also possible to store only the results in memory, without defining neither a filename nor the "save" tag. @@ -431,7 +431,7 @@ Access Protocol (TAP) and via the Astronomical Data Query Language (ADQL). >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> result = esahubble.query_hst_tap("select top 10 * from hsc_v2.hubble_sc2", "test.vot.gz") + >>> result = esahubble.query_tap("select top 10 * from hsc_v2.hubble_sc2", "test.vot.gz") INFO: Query finished. [astroquery.utils.tap.core] This will execute an ADQL query to download the first 10 sources in the From ac39c9c8fe50fad01318fbc8d9e6230c792f1318 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:11:11 +0100 Subject: [PATCH 02/14] Fix for #2590 --- astroquery/esa/jwst/core.py | 7 ++--- astroquery/esa/jwst/tests/test_jwstdata.py | 2 +- astroquery/esa/jwst/tests/test_jwsttap.py | 31 +++++++++++++--------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 7342ed068e..f333ea5fa2 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -8,6 +8,7 @@ European Space Agency (ESA) """ + import binascii import gzip import os @@ -60,7 +61,7 @@ class JwstClass(BaseQuery): def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=True): if tap_plus_handler is None: self.__jwsttap = TapPlus(url=conf.JWST_TAP_SERVER, - data_context='data') + data_context='data', client_id='ASTROQUERY') else: self.__jwsttap = tap_plus_handler @@ -887,7 +888,7 @@ def get_product(self, *, artifact_id=None, file_name=None): params_dict = {} params_dict['RETRIEVAL_TYPE'] = 'PRODUCT' - params_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + params_dict['TAPCLIENT'] = 'ASTROQUERY' self.__check_product_input(artifact_id=artifact_id, file_name=file_name) @@ -981,7 +982,7 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", params_dict = {} params_dict['RETRIEVAL_TYPE'] = 'OBSERVATION' - params_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + params_dict['TAPCLIENT'] = 'ASTROQUERY' plane_ids = self._get_associated_planes(plane_ids=plane_ids, cal_level=cal_level, diff --git a/astroquery/esa/jwst/tests/test_jwstdata.py b/astroquery/esa/jwst/tests/test_jwstdata.py index 609372741b..f5c7a64792 100644 --- a/astroquery/esa/jwst/tests/test_jwstdata.py +++ b/astroquery/esa/jwst/tests/test_jwstdata.py @@ -50,7 +50,7 @@ def test_get_product(self): parameters = {} params_dict = {} params_dict['RETRIEVAL_TYPE'] = 'PRODUCT' - params_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + params_dict['TAPCLIENT'] = 'ASTROQUERY' params_dict['ARTIFACTID'] = '00000000-0000-0000-8740-65e2827c9895' parameters['params_dict'] = params_dict parameters['output_file'] = 'jw00617023001_02102_00001_nrcb4_uncal.fits' diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 7c58e2d085..be7ede0b9a 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -19,6 +19,7 @@ from astropy.coordinates.sky_coordinate import SkyCoord from astropy.table import Table from astropy.units import Quantity +from astroquery.exceptions import TableParseError from astroquery.esa.jwst import JwstClass from astroquery.esa.jwst.tests.DummyTapHandler import DummyTapHandler @@ -594,7 +595,7 @@ def test_get_product_by_artifactid(self): param_dict = {} param_dict['RETRIEVAL_TYPE'] = 'PRODUCT' - param_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + param_dict['TAPCLIENT'] = 'ASTROQUERY' param_dict['ARTIFACTID'] = '00000000-0000-0000-8740-65e2827c9895' parameters['params_dict'] = param_dict @@ -618,7 +619,7 @@ def test_get_product_by_filename(self): param_dict = {} param_dict['RETRIEVAL_TYPE'] = 'PRODUCT' - param_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + param_dict['TAPCLIENT'] = 'ASTROQUERY' param_dict['ARTIFACTID'] = '00000000-0000-0000-8740-65e2827c9895' parameters['params_dict'] = param_dict @@ -682,7 +683,7 @@ def test_get_obs_products(self): param_dict = {} param_dict['RETRIEVAL_TYPE'] = 'OBSERVATION' - param_dict['DATA_RETRIEVAL_ORIGIN'] = 'ASTROQUERY' + param_dict['TAPCLIENT'] = 'ASTROQUERY' param_dict['planeid'] = planeids param_dict['calibrationlevel'] = 'ALL' parameters['params_dict'] = param_dict @@ -889,13 +890,14 @@ def test_query_target_error(self): ned = Ned() vizier = Vizier() # Testing default parameters - with pytest.raises(ValueError) as err: + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="M1", target_resolver="") assert "This target resolver is not allowed" in err.value.args[0] - with pytest.raises(ValueError) as err: + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target("TEST") - assert "This target name cannot be determined with this resolver: ALL" in err.value.args[0] - with pytest.raises(ValueError) as err: + assert "This target name cannot be determined with this resolver: ALL" in err.value.args[0] or "Failed " + "to parse" in err.value.args[0] + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="M1", target_resolver="ALL") assert err.value.args[0] in ["This target name cannot be determined " "with this resolver: ALL", "Missing " @@ -913,20 +915,23 @@ def test_query_target_error(self): vizier.query_object = MagicMock(return_value=vizier_table) # coordinate_error = 'coordinate must be either a string or astropy.coordinates' - with pytest.raises(ValueError) as err: + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="SIMBAD", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: SIMBAD' in err.value.args[0] + assert 'This target name cannot be determined with this resolver: SIMBAD' in err.value.args[0] or "Failed " + "to parse" in err.value.args[0] - with pytest.raises(ValueError) as err: + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="NED", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: NED' in err.value.args[0] + assert 'This target name cannot be determined with this resolver: NED' in err.value.args[0] or "Failed " + "to parse" in err.value.args[0] - with pytest.raises(ValueError) as err: + with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="VIZIER", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: VIZIER' in err.value.args[0] + assert 'This target name cannot be determined with this resolver: VIZIER' in err.value.args[0] or "Failed " + "to parse" in err.value.args[0] def test_remove_jobs(self): dummyTapHandler = DummyTapHandler() From 462ea9b01241a71ebce7be3f05ad439705a1148f Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:11:50 +0100 Subject: [PATCH 03/14] Custom client_id in TapPlus --- astroquery/utils/tap/core.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index cf14219c1d..87b9a9aabf 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -34,7 +34,7 @@ __all__ = ['Tap', 'TapPlus'] VERSION = "20200428.1" -TAP_CLIENT_ID = f"aqtappy-{VERSION}" +TAP_CLIENT_ID = f"aqtappy1-{VERSION}" class Tap: @@ -745,7 +745,8 @@ def __init__(self, url=None, table_edit_context=None, data_context=None, datalink_context=None, - verbose=False): + verbose=False, + client_id=None): """Constructor Parameters @@ -791,12 +792,19 @@ def __init__(self, url=None, connhandler=connhandler, verbose=verbose) self.__internalInit() + self.__set_client_id(client_id=client_id) def __internalInit(self): self.__user = None self.__pwd = None self.__isLoggedIn = False + @staticmethod + def __set_client_id(client_id): + if client_id: + global TAP_CLIENT_ID + TAP_CLIENT_ID = client_id + def load_tables(self, only_names=False, include_shared_tables=False, verbose=False): """Loads all public tables From 023eaec74db7721c865d857d9f1026502e5d45fe Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:12:02 +0100 Subject: [PATCH 04/14] Fixes in remote tests --- docs/esa/iso/iso.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/esa/iso/iso.rst b/docs/esa/iso/iso.rst index 890dd00754..3658e12c49 100644 --- a/docs/esa/iso/iso.rst +++ b/docs/esa/iso/iso.rst @@ -97,13 +97,14 @@ provided by this service, see section 'Getting Tables Details'. '80000203.tar' >>> tar = tarfile.open("80000203.tar") >>> tar.list() # doctest: +IGNORE_OUTPUT - -rw-r--r-- idaops/0 1094 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//pich80000203.gif - -rw-r--r-- idaops/0 266240 2005-12-23 11:02:54 ././ISO1659972236/EXTRAKON//C10480000203.tar - -rw-r--r-- idaops/0 14400 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//psph80000203.fits - -rw-r--r-- idaops/0 5599 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//ppch80000203.gif - -rw-r--r-- idaops/0 266240 2005-12-23 11:02:54 ././ISO1659972236/EXTRAKON//C10180000203.tar + -rw-r--r-- idaops/0 1094 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//pich80000203.gif + -rw-r--r-- idaops/0 266240 2005-12-23 11:02:54 ././ISO1659972236/EXTRAKON//C10480000203.tar + -rw-r--r-- idaops/0 14400 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//psph80000203.fits + -rw-r--r-- idaops/0 5599 2005-12-23 11:02:55 ././ISO1659972236/EXTRAKON//ppch80000203.gif + -rw-r--r-- idaops/0 266240 2005-12-23 11:02:54 ././ISO1659972236/EXTRAKON//C10180000203.tar >>> tar.extract("././ISO1659972236/EXTRAKON//psph80000203.fits") # doctest: +SKIP >>> tar.extractall() + >>> tar.close() 'download_data' method invokes the data download of files from the ISO Data Archive, using the observation identifier (TDT) as input. There are different product levels: @@ -134,8 +135,8 @@ we can loop on the observations to download the best products (DEFAULT_DATA_SET) ... "ida.observations WHERE " + ... "revno=800 and aotname like 'S%' and target != ''") >>> print(table) # doctest: +IGNORE_OUTPUT - tdt revno aotname ra dec - h deg + tdt revno aotname ra dec + h deg -------- ----- ------- ------------ -------- 80002247 800 S07 2.428012666 62.09789 80002014 800 S02 0.901055333 73.08528 @@ -211,7 +212,7 @@ This will show the available tables in ISO TAP service in the ISO Data Archive. .. doctest-remote-data:: - >>> from astroquery.esa.iso import ISO + >>> from astroquery.esa.iso import ISO >>> ISO.get_columns('ida.observations') INFO: Retrieving tables... [astroquery.utils.tap.core] INFO: Parsing tables... [astroquery.utils.tap.core] @@ -369,7 +370,7 @@ And spectra can be displayed by using the following code: ... 'RETRIEVAL_TYPE=STANDALONE&obsno=580020010', ... '58002102.fits') # doctest: +IGNORE_OUTPUT ('58002102.fits', ) - >>> # Opening the spectral fits file using astropy modules + >>> # Opening the spectral fits file using astropy modules >>> quantity_support() .MplQuantityConverter object at 0x11c1a9d60> >>> f = fits.open('58002102.fits') From 22a15e71f11d16066940ac35fe4e23fdee535253 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:39:43 +0100 Subject: [PATCH 05/14] EHSTPCR-1071: style fixes --- astroquery/esa/hubble/core.py | 8 +------- astroquery/esa/hubble/tests/test_esa_hubble_remote.py | 2 -- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index d4e976aa7a..58a76e1a61 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -13,15 +13,9 @@ from astropy.coordinates import SkyCoord from astropy.coordinates import Angle -from requests.exceptions import ConnectionError - -from astroquery.exceptions import RemoteServiceError from astroquery.utils.tap import TapPlus -from astroquery.utils import commons from astroquery.query import BaseQuery -import shutil import json -from urllib.parse import urlencode import warnings from astropy.utils.exceptions import AstropyDeprecationWarning @@ -100,7 +94,7 @@ def download_product(self, observation_id, *, calibration_level=None, params["PRODUCTTYPE"] = product_type filename = self._get_product_filename(product_type, filename) - response = self._tap.load_data(params, filename, verbose=verbose) + self._tap.load_data(params, filename, verbose=verbose) return filename diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index 3da24bbbfd..2a7f04250f 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -17,7 +17,6 @@ from astroquery.esa.hubble import ESAHubble from astropy import coordinates import random -from astropy.table import Table esa_hubble = ESAHubble() @@ -76,7 +75,6 @@ def test_get_artifact(self): def test_cone_search(self): esa_hubble = ESAHubble() c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - temp_file = self.temp_folder.name + "/cone_search_m31_5.vot" compressed_temp_file = self.temp_folder.name + "/cone_search_m31_5.vot.gz" # open & extracting the file table = esa_hubble.cone_search(c, 7, compressed_temp_file, verbose=True) From 7aa111c0bda365cca279b7ac2a68d7e2ba35e69e Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 16:56:21 +0100 Subject: [PATCH 06/14] changes.rst file updated --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index d7172589e5..23b00b1c38 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -32,7 +32,7 @@ esa.hubble - Refactored query_criteria to use ehst.archive table therefore making the query a lot faster. [#2524] -- Update TAP url to avoid 301 HTTPError. [#2567] +- Update to TAP url to query data and download files, aligned with the new eHST Science Archive. [#2567][#2597] alma From 718012624f29164cd5192ce4cd124cbb70329809 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Wed, 23 Nov 2022 17:26:02 +0100 Subject: [PATCH 07/14] EHSTPCR-1071: fix in test cov --- astroquery/esa/hubble/tests/test_esa_hubble.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index a5d8ea1245..eb6f03c36a 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -130,6 +130,9 @@ def test_get_postcard(self, tmp_path): ehst.get_postcard(observation_id="X0MC5101T", filename=Path(tmp_path, "X0MC5101T.vot"), verbose=True) + ehst.get_postcard(observation_id="X0MC5101T", + filename=Path(tmp_path, "X0MC5101T.vot"), resolution=1024, + verbose=True) @patch.object(ESAHubbleClass, 'cone_search') @patch.object(ESAHubbleClass, '_query_tap_target') From 0001244032b85d0ad112ea9563d17acc0d4283b5 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 11:47:22 +0100 Subject: [PATCH 08/14] eHST: tests and doc updated, kwargs methods --- CHANGES.rst | 11 +- astroquery/esa/hubble/core.py | 32 +++- .../esa/hubble/tests/test_esa_hubble.py | 165 ++++++++++-------- .../hubble/tests/test_esa_hubble_remote.py | 21 +-- docs/esa/hubble/hubble.rst | 57 ++++-- 5 files changed, 177 insertions(+), 109 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 23b00b1c38..2d7c44689d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,12 @@ hsa - New module to access ESA Herschel mission. [#2122] +esa.hubble +^^^^^^^^^^ + +- Update to TAP url to query data and download files, aligned with the new eHST Science Archive. [#2567][#2597] +- Status and maintenance messages from eHST TAP when the module is instantiated. get_status_messages method to retrieve them. [#2597] +- Optional parameters in all methods are kwargs keyword only. [#2597] Service fixes and enhancements ------------------------------ @@ -31,9 +37,8 @@ esa.hubble - Refactored query_criteria to use ehst.archive table therefore making the query a lot faster. [#2524] - -- Update to TAP url to query data and download files, aligned with the new eHST Science Archive. [#2567][#2597] - +- Method query_hst_tap has been deprecated and is replaced with query_tap, with the same arguments. [#2597] +- Product types in download_product method have been modified to: PRODUCT, SCIENCE_PRODUCT or POSTCARD. [#2597] alma ^^^^ diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index 58a76e1a61..d0e9f1570b 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -18,6 +18,7 @@ import json import warnings from astropy.utils.exceptions import AstropyDeprecationWarning +from astropy.utils.decorators import deprecated from . import conf from astroquery import log @@ -44,6 +45,7 @@ def __init__(self, *, tap_handler=None, show_messages=True): if show_messages: self.get_status_messages() + def download_product(self, observation_id, *, calibration_level=None, filename=None, verbose=False, product_type=None): """ @@ -71,11 +73,20 @@ def download_product(self, observation_id, *, calibration_level=None, product_type : string type of product retrieval, optional SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY + ------------ + Deprecation Warning: PRODUCT, SCIENCE_PRODUCT or POSTCARD + are no longer supported. + ------------ Returns ------- None. It downloads the observation indicated """ + if product_type and product_type in ['PRODUCT', 'SCIENCE_PRODUCT', 'POSTCARD']: + warnings.warn( + "PRODUCT, SCIENCE_PRODUCT or POSTCARD product types are no longer supported. " + "Please use SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY instead.", + AstropyDeprecationWarning) params = {"OBSERVATIONID": observation_id, "TAPCLIENT": "ASTROQUERY", @@ -217,7 +228,7 @@ def _get_product_filename(self, product_type, filename): return filename - def get_artifact(self, artifact_id, filename=None, verbose=False): + def get_artifact(self, artifact_id, *, filename=None, verbose=False): """ Download artifacts from EHST. Artifact is a single Hubble product file. @@ -302,7 +313,7 @@ def __get_product_type_by_resolution(self, resolution): else: raise ValueError("Resolution must be 256 or 1024") - def cone_search(self, coordinates, radius, filename=None, + def cone_search(self, coordinates, radius, *, filename=None, output_format='votable', cache=True, async_job=False, verbose=False): """ @@ -371,7 +382,7 @@ def cone_search(self, coordinates, radius, filename=None, verbose=verbose) return table - def cone_search_criteria(self, radius, target=None, + def cone_search_criteria(self, radius, *, target=None, coordinates=None, calibration_level=None, data_product_type=None, @@ -503,7 +514,7 @@ def _query_tap_target(self, target): except (ValueError, KeyError): raise ValueError("This target cannot be resolved") - def query_metadata(self, output_format='votable', verbose=False): + def query_metadata(self, *, output_format='votable', verbose=False): return def query_target(self, name, *, filename=None, output_format='votable', @@ -540,7 +551,7 @@ def query_target(self, name, *, filename=None, output_format='votable', return table - def query_tap(self, query, async_job=False, output_file=None, + def query_tap(self, query, *, async_job=False, output_file=None, output_format="votable", verbose=False): """Launches a synchronous or asynchronous job to query the HST tap @@ -577,7 +588,10 @@ def query_tap(self, query, async_job=False, output_file=None, table = job.get_results() return table - def query_hst_tap(self, query, async_job=False, output_file=None, + @deprecated(since="0.4.7", message=("Use of query_hst_tap method is no longer supported. " + "Please use query_tap method instead, with the same arguments."), + alternative="query_tap") + def query_hst_tap(self, query, *, async_job=False, output_file=None, output_format="votable", verbose=False): """Launches a synchronous or asynchronous job to query the HST tap @@ -607,7 +621,7 @@ def query_hst_tap(self, query, async_job=False, output_file=None, self.query_tap(query=query, async_job=False, output_file=None, output_format="votable", verbose=False) - def query_criteria(self, calibration_level=None, + def query_criteria(self, *, calibration_level=None, data_product_type=None, intent=None, obs_collection=None, instrument_name=None, filters=None, async_job=True, output_file=None, @@ -720,7 +734,7 @@ def __check_list_strings(self, list): raise ValueError("One of the lists is empty or there are " "elements that are not strings") - def get_tables(self, only_names=True, verbose=False): + def get_tables(self, *, only_names=True, verbose=False): """Get the available table in EHST TAP service Parameters @@ -762,7 +776,7 @@ def get_status_messages(self): except OSError: print("Status messages could not be retrieved") - def get_columns(self, table_name, only_names=True, verbose=False): + def get_columns(self, table_name, *, only_names=True, verbose=False): """Get the available columns for a table in EHST TAP service Parameters diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index eb6f03c36a..29e9713ebe 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -23,6 +23,8 @@ from astroquery.esa.hubble import ESAHubbleClass from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler +from astroquery.utils.tap.conn.tapconn import ConnectionHandler +from astropy.utils.exceptions import AstropyDeprecationWarning def data_path(filename): @@ -49,10 +51,10 @@ def ehst_request(request): return mp -def get_cone_mockreturn(method, request, url, params, *args, **kwargs): +def get_cone_mockreturn(params, *args, **kwargs): file = data_path('cone_search_m31_5.vot') - if 'OBSERVATION_ID' in params: - file = params['OBSERVATION_ID'] + ".vot" + if 'OBSERVATION_ID' in kwargs: + file = kwargs['OBSERVATION_ID'] + ".vot" response = data_path(file) shutil.copy(response + '.test', response) return response @@ -104,7 +106,7 @@ def test_download_product_by_calibration(self, tmp_path): def test_download_product_by_product_type(self, tmp_path): parameters = {'observation_id': "J6FL25S4Q", - 'product_type': "SCIENCE_PRODUCT", + 'product_type': "SCIENCE", 'filename': Path(tmp_path, "J6FL25S4Q.vot.test"), 'verbose': True} ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) @@ -112,13 +114,13 @@ def test_download_product_by_product_type(self, tmp_path): product_type=parameters['product_type'], filename=parameters['filename'], verbose=parameters['verbose']) - parameters['product_type'] = "PRODUCT" + parameters['product_type'] = "SCIENCE" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], product_type=parameters['product_type'], filename=parameters['filename'], verbose=parameters['verbose']) - parameters['product_type'] = "POSTCARD" + parameters['product_type'] = "PREVIEW" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.download_product(observation_id=parameters['observation_id'], product_type=parameters['product_type'], @@ -158,11 +160,11 @@ def test_cone_search(self): response = Response() response._content = target_obj ehst._request = MagicMock(return_value=response) - ehst.cone_search(parameters['coordinates'], - parameters['radius'], - parameters['filename'], - parameters['output_format'], - parameters['cache']) + ehst.cone_search(coordinates=parameters['coordinates'], + radius=parameters['radius'], + filename=parameters['filename'], + output_format=parameters['output_format'], + cache=parameters['cache']) DummyHubbleTapHandler("cone_search", parameters) def test_cone_search_coords(self): @@ -183,13 +185,13 @@ def test_cone_search_coords(self): 'verbose': True} ehst = ESAHubbleClass(tap_handler=dummyTapHandler, show_messages=False) - ehst.cone_search(parameters['coordinates'], - parameters['radius'], - parameters['filename'], - parameters['output_format'], - parameters['async_job'], - parameters['cache'], - parameters['verbose']) + ehst.cone_search(coordinates=parameters['coordinates'], + radius=parameters['radius'], + filename=parameters['filename'], + output_format=parameters['output_format'], + async_job=parameters['async_job'], + cache=parameters['cache'], + verbose=parameters['verbose']) with pytest.raises(ValueError) as err: ehst._getCoordInput(1234) assert "Coordinates must be either a string or " \ @@ -207,9 +209,11 @@ def test_query_tap(self): 'verbose': False} ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.query_tap(parameters['query'], parameters['async_job'], - parameters['output_file'], - parameters['output_format'], parameters['verbose']) + ehst.query_tap(query=parameters['query'], + async_job=parameters['async_job'], + output_file=parameters['output_file'], + output_format=parameters['output_format'], + verbose=parameters['verbose']) self.get_dummy_tap_handler().check_call("launch_job", parameters2) def test_get_tables(self): @@ -218,12 +222,12 @@ def test_get_tables(self): DummyHubbleTapHandler("get_tables", parameters) ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_tables(True, True) + ehst.get_tables(only_names=True, verbose=True) def test_get_artifact(self, tmp_path): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) path = Path(tmp_path, "w0ji0v01t_c2f.fits.gz") - ehst.get_artifact(path) + ehst.get_artifact(artifact_id=path) def test_get_columns(self): parameters = {'table_name': "table", @@ -232,7 +236,7 @@ def test_get_columns(self): dummyTapHandler = DummyHubbleTapHandler("get_columns", parameters) ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_columns("table", True, True) + ehst.get_columns(table_name="table", only_names=True, verbose=True) dummyTapHandler.check_call("get_columns", parameters) def test_query_criteria(self): @@ -248,17 +252,17 @@ def test_query_criteria(self): 'verbose': True, 'get_query': True} ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - test_query = ehst.query_criteria(parameters1['calibration_level'], - parameters1['data_product_type'], - parameters1['intent'], - parameters1['obs_collection'], - parameters1['instrument_name'], - parameters1['filters'], - parameters1['async_job'], - parameters1['output_file'], - parameters1['output_format'], - parameters1['verbose'], - parameters1['get_query']) + test_query = ehst.query_criteria(calibration_level=parameters1['calibration_level'], + data_product_type=parameters1['data_product_type'], + intent=parameters1['intent'], + obs_collection=parameters1['obs_collection'], + instrument_name=parameters1['instrument_name'], + filters=parameters1['filters'], + async_job=parameters1['async_job'], + output_file=parameters1['output_file'], + output_format=parameters1['output_format'], + verbose=parameters1['verbose'], + get_query=parameters1['get_query']) parameters2 = {'query': test_query, 'output_file': "output_test_query_by_criteria.vot.gz", 'output_format': "votable", @@ -289,17 +293,17 @@ def test_query_criteria_numeric_calibration(self): 'verbose': True, 'get_query': True} ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - test_query = ehst.query_criteria(parameters1['calibration_level'], - parameters1['data_product_type'], - parameters1['intent'], - parameters1['obs_collection'], - parameters1['instrument_name'], - parameters1['filters'], - parameters1['async_job'], - parameters1['output_file'], - parameters1['output_format'], - parameters1['verbose'], - parameters1['get_query']) + test_query = ehst.query_criteria(calibration_level=parameters1['calibration_level'], + data_product_type=parameters1['data_product_type'], + intent=parameters1['intent'], + obs_collection=parameters1['obs_collection'], + instrument_name=parameters1['instrument_name'], + filters=parameters1['filters'], + async_job=parameters1['async_job'], + output_file=parameters1['output_file'], + output_format=parameters1['output_format'], + verbose=parameters1['verbose'], + get_query=parameters1['get_query']) parameters2 = {'query': test_query, 'output_file': "output_test_query_by_criteria.vot.gz", 'output_format': "votable", @@ -318,17 +322,17 @@ def test_query_criteria_numeric_calibration(self): dummy_tap_handler.check_call("launch_job", parameters3) parameters1['calibration_level'] = 4 with pytest.raises(KeyError) as err: - ehst.query_criteria(parameters1['calibration_level'], - parameters1['data_product_type'], - parameters1['intent'], - parameters1['obs_collection'], - parameters1['instrument_name'], - parameters1['filters'], - parameters1['async_job'], - parameters1['output_file'], - parameters1['output_format'], - parameters1['verbose'], - parameters1['get_query']) + ehst.query_criteria(calibration_level=parameters1['calibration_level'], + data_product_type=parameters1['data_product_type'], + intent=parameters1['intent'], + obs_collection=parameters1['obs_collection'], + instrument_name=parameters1['instrument_name'], + filters=parameters1['filters'], + async_job=parameters1['async_job'], + output_file=parameters1['output_file'], + output_format=parameters1['output_format'], + verbose=parameters1['verbose'], + get_query=parameters1['get_query']) assert "Calibration level must be between 0 and 3" in err.value.args[0] def test_cone_search_criteria(self): @@ -471,7 +475,7 @@ def test__select_related_composite(self, mock_query): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" - oids = ehst._select_related_composite(dummy_obs_id) + oids = ehst._select_related_composite(observation_id=dummy_obs_id) assert oids == ['x', 'y'] @patch.object(ESAHubbleClass, 'query_tap') @@ -483,7 +487,7 @@ def test__select_related_members(self, mock_query): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" - oids = ehst._select_related_members(dummy_obs_id) + oids = ehst._select_related_members(observation_id=dummy_obs_id) assert oids == ['test'] @patch.object(ESAHubbleClass, 'query_tap') @@ -495,14 +499,14 @@ def test_get_observation_type(self, mock_query): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" - oids = ehst.get_observation_type(dummy_obs_id) + oids = ehst.get_observation_type(observation_id=dummy_obs_id) assert oids == 'HST Test' def test_get_observation_type_obs_id_none_valueerror(self): with pytest.raises(ValueError): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = None - ehst.get_observation_type(dummy_obs_id) + ehst.get_observation_type(observation_id=dummy_obs_id) @patch.object(ESAHubbleClass, 'query_tap') def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query): @@ -514,7 +518,7 @@ def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = '1234' - ehst.get_observation_type(dummy_obs_id) + ehst.get_observation_type(observation_id=dummy_obs_id) @patch.object(ESAHubbleClass, 'query_tap') @patch.object(ESAHubbleClass, 'get_observation_type') @@ -527,7 +531,7 @@ def test_get_hst_link(self, mock_observation_type, mock_query): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" - oids = ehst.get_hap_hst_link(dummy_obs_id) + oids = ehst.get_hap_hst_link(observation_id=dummy_obs_id) assert oids == ['1234'] @patch.object(ESAHubbleClass, 'get_observation_type') @@ -537,7 +541,7 @@ def test_get_hap_link(self, mock_select_related_members, mock_observation_type): mock_observation_type.return_value = "HAP" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" - oids = ehst.get_hap_hst_link(dummy_obs_id) + oids = ehst.get_hap_hst_link(observation_id=dummy_obs_id) assert oids == 'test' @patch.object(ESAHubbleClass, 'get_observation_type') @@ -546,7 +550,7 @@ def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): mock_observation_type.return_value = "valueerror" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" - ehst.get_hap_hst_link(dummy_obs_id) + ehst.get_hap_hst_link(observation_id=dummy_obs_id) @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_hap_hst_link_compositeerror(self, mock_observation_type): @@ -554,7 +558,7 @@ def test_get_hap_hst_link_compositeerror(self, mock_observation_type): mock_observation_type.return_value = "HAP Composite" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" - ehst.get_hap_hst_link(dummy_obs_id) + ehst.get_hap_hst_link(observation_id=dummy_obs_id) @patch.object(ESAHubbleClass, '_select_related_members') @patch.object(ESAHubbleClass, 'get_observation_type') @@ -563,7 +567,7 @@ def test_get_member_observations_composite(self, mock_observation_type, mock_sel ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_select_related_members.return_value = 'test' dummy_obs_id = "1234" - oids = ehst.get_member_observations(dummy_obs_id) + oids = ehst.get_member_observations(observation_id=dummy_obs_id) assert oids == 'test' @patch.object(ESAHubbleClass, '_select_related_composite') @@ -573,7 +577,7 @@ def test_get_member_observations_simple(self, mock_observation_type, mock_select ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) mock_select_related_composite.return_value = 'test' dummy_obs_id = "1234" - oids = ehst.get_member_observations(dummy_obs_id) + oids = ehst.get_member_observations(observation_id=dummy_obs_id) assert oids == 'test' @patch.object(ESAHubbleClass, 'get_observation_type') @@ -582,7 +586,7 @@ def test_get_member_observations_invalid_id_valueerror(self, mock_observation_ty mock_observation_type.return_value = "valueerror" ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) dummy_obs_id = "1234" - ehst.get_member_observations(dummy_obs_id) + ehst.get_member_observations(observation_id=dummy_obs_id) @patch.object(ESAHubbleClass, 'query_criteria') @patch.object(ESAHubbleClass, '_query_tap_target') @@ -610,3 +614,26 @@ def test_cone_search_criteria_typeerror(self, mock_query_criteria): with pytest.raises(TypeError): ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", target="m11", radius=1) + + def test_query_hst_tap(self): + parameters = {'query': "select top 10 * from hsc_v2.hubble_sc2", + 'async_job': False, + 'output_file': "test2.vot", + 'output_format': "votable", + 'verbose': False} + + ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + with pytest.warns(AstropyDeprecationWarning): + ehst.query_hst_tap(query=parameters['query'], + async_job=parameters['async_job'], + output_file=parameters['output_file'], + output_format=parameters['output_format'], + verbose=parameters['verbose']) + + @patch("http.client.HTTPSConnection") + @patch("http.client.HTTPResponse") + def test_show_messages(self, mock_conn, mock_res): + mock_res.status = 400 + mock_conn.getresponse = MagicMock(return_value=mock_res) + ehst = ESAHubbleClass() + mock_res.assert_called() diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index 2a7f04250f..f3262b084e 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -12,6 +12,7 @@ import tempfile import os +from unittest.mock import patch, MagicMock import pytest from astroquery.esa.hubble import ESAHubble @@ -51,13 +52,13 @@ class TestEsaHubbleRemoteData: temp_folder = create_temp_folder() def test_query_tap_async(self): - result = esa_hubble.query_tap(self.top_obs_query, async_job=True) + result = esa_hubble.query_tap(query=self.top_obs_query, async_job=True) assert len(result) > 10 assert "observation_id" in result.keys() remove_last_job() def test_download_product(self): - result = esa_hubble.query_tap(self.hst_query) + result = esa_hubble.query_tap(query=self.hst_query) observation_id = random.choice(result['observation_id']) temp_file = self.temp_folder.name + "/" + observation_id + ".tar" esa_hubble.download_product(observation_id=observation_id, @@ -65,11 +66,11 @@ def test_download_product(self): assert os.path.exists(temp_file) def test_get_artifact(self): - result = esa_hubble.query_tap(self.top_artifact_query) + result = esa_hubble.query_tap(query=self.top_artifact_query) assert "artifact_id" in result.keys() artifact_id = random.choice(result["artifact_id"]) temp_file = self.temp_folder.name + "/" + artifact_id + ".gz" - esa_hubble.get_artifact(artifact_id, temp_file) + esa_hubble.get_artifact(artifact_id=artifact_id, filename=temp_file) assert os.path.exists(temp_file) def test_cone_search(self): @@ -77,7 +78,7 @@ def test_cone_search(self): c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') compressed_temp_file = self.temp_folder.name + "/cone_search_m31_5.vot.gz" # open & extracting the file - table = esa_hubble.cone_search(c, 7, compressed_temp_file, verbose=True) + table = esa_hubble.cone_search(coordinates=c, radius=7, filename=compressed_temp_file, verbose=True) assert 'observation_id' in table.columns assert len(table) > 0 remove_last_job() @@ -90,27 +91,27 @@ def test_hst_composite_to_hst_simple(self): def test_hst_simple_to_hst_composite(self): esa_hubble = ESAHubble() - result = esa_hubble.get_member_observations('jdrz0cjxq') + result = esa_hubble.get_member_observations(observation_id='jdrz0cjxq') assert result == ['jdrz0c010'] def test_hap_composite_to_hap_simple(self): esa_hubble = ESAHubble() - result = esa_hubble.get_member_observations('hst_15446_4v_acs_wfc_f606w_jdrz4v') + result = esa_hubble.get_member_observations(observation_id='hst_15446_4v_acs_wfc_f606w_jdrz4v') assert result == ['hst_15446_4v_acs_wfc_f606w_jdrz4vkv', 'hst_15446_4v_acs_wfc_f606w_jdrz4vkw'] def test_hap_simple_to_hap_composite(self): esa_hubble = ESAHubble() - result = esa_hubble.get_member_observations('hst_16316_71_acs_sbc_f150lp_jec071i9') + result = esa_hubble.get_member_observations(observation_id='hst_16316_71_acs_sbc_f150lp_jec071i9') assert result == [' hst_16316_71_acs_sbc_total_jec071', 'hst_16316_71_acs_sbc_f150lp_jec071'] def test_hap_simple_to_hst_simple(self): esa_hubble = ESAHubble() - result = esa_hubble.get_hap_hst_link('hst_16316_71_acs_sbc_f150lp_jec071i9') + result = esa_hubble.get_hap_hst_link(observation_id='hst_16316_71_acs_sbc_f150lp_jec071i9') assert result == ['jec071i9q'] def test_hst_simple_to_hap_simple(self): esa_hubble = ESAHubble() - result = esa_hubble.get_hap_hst_link('jec071i9q') + result = esa_hubble.get_hap_hst_link(observation_id='jec071i9q') assert result == ['hst_16316_71_acs_sbc_f150lp_jec071i9'] def test_query_target(self): diff --git a/docs/esa/hubble/hubble.rst b/docs/esa/hubble/hubble.rst index 76d9172774..6d4d5506f2 100644 --- a/docs/esa/hubble/hubble.rst +++ b/docs/esa/hubble/hubble.rst @@ -23,13 +23,26 @@ data, all HST data in the EHST are identical to those in MAST. Examples ======== +It is highly recommended checking the status of eHST TAP before executing this module. To do this: + +.. doctest-remote-data:: + + >>> from astroquery.esa.hubble import ESAHubble + >>> esahubble = ESAHubble() + >>> esahubble.get_status_messages() + +This method will retrieve the same warning messages shown in eHST Science Archive with information about +service degradation. + -------------------------- 1. Getting Hubble products -------------------------- This function allows the user to download products based on their observation ID (mandatory) and -a required calibration_level (RAW, CALIBRATED, PRODUCT or AUXILIARY) and/or product type (PRODUCT, -SCIENCE_PRODUCT or POSTCARD). +a required calibration_level (RAW, CALIBRATED, PRODUCT or AUXILIARY) and/or product type (SCIENCE, PREVIEW, THUMBNAIL or AUXILIARY). + +Deprecation Warning: product types PRODUCT, SCIENCE_PRODUCT or POSTCARD are no longer supported. Please modify your scripts accordingly. + This will download all files for the raw calibration level of the observation 'j6fl25s4q' and it will store them in a tar called 'raw_data_for_j6fl25s4q.tar'. @@ -48,7 +61,7 @@ This will download the science files associated to the observation 'j6fl25s4q' a >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.download_product(observation_id="j6fl25s4q", product_type="SCIENCE_PRODUCT", + >>> esahubble.download_product(observation_id="j6fl25s4q", product_type="SCIENCE", ... filename="science_data_for_j6fl25s4q.fits") # doctest: +IGNORE_OUTPUT This third case will download the science files associated to the observation 'j6fl25s4q' in raw calibration level and it will store them in a file called @@ -59,7 +72,7 @@ This third case will download the science files associated to the observation 'j >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() >>> esahubble.download_product(observation_id="j6fl25s4q", calibration_level="RAW", - ... filename="science_raw_data_for_j6fl25s4q", product_type="SCIENCE_PRODUCT") # doctest: +IGNORE_OUTPUT + ... filename="science_raw_data_for_j6fl25s4q", product_type="SCIENCE") # doctest: +IGNORE_OUTPUT --------------------------- 2. Getting Hubble postcards @@ -88,7 +101,7 @@ Note: Artifact is a single Hubble product file. >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> esahubble.get_artifact("w0ji0v01t_c2f.fits") + >>> esahubble.get_artifact(artifact_id="w0ji0v01t_c2f.fits") This will download the compressed artifact 'w0ji0v01t_c2f.fits.gz'. 'w0ji0v01t_c2f.fits' is the name of the Hubble @@ -104,7 +117,7 @@ The query_target function queries the name of the target as given by the propose >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> table = esahubble.query_target("m31", filename="m31_query.xml.gz") # doctest: +IGNORE_OUTPUT + >>> table = esahubble.query_target(name="m31", filename="m31_query.xml.gz") # doctest: +IGNORE_OUTPUT This will retrieve a table with the output of the query. It will also download a file storing all metadata for all observations @@ -265,15 +278,15 @@ As has been mentioned, these parameters are optional and it is not necessary to >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() >>> result1 = esahubble.query_criteria(calibration_level = 'PRODUCT', - ... async_job = False, - ... output_file = 'output3.vot.gz') + ... async_job = False, + ... output_file = 'output3.vot.gz') >>> result2 = esahubble.query_criteria(data_product_type = 'image', - ... intent='SCIENCE', - ... async_job = False, - ... output_file = 'output4.vot.gz') + ... intent='SCIENCE', + ... async_job = False, + ... output_file = 'output4.vot.gz') >>> result3 = esahubble.query_criteria(data_product_type = 'timeseries', - ... async_job = False, - ... output_file = 'output5.vot.gz') + ... async_job = False, + ... output_file = 'output5.vot.gz') If no criteria are specified to limit the selection, this function will retrieve all the observations. @@ -319,7 +332,7 @@ This last example will provide the ADQL query based on the criteria defined by t >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() >>> c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - >>> table = esahubble.cone_search(c, 7, "cone_search_m31_5.vot.gz") + >>> table = esahubble.cone_search(coordinates=c, radius=7, filename="cone_search_m31_5.vot.gz") This will perform a cone search with radius 7 arcmins. The result of the query will be returned and stored in the votable file @@ -431,8 +444,7 @@ Access Protocol (TAP) and via the Astronomical Data Query Language (ADQL). >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> result = esahubble.query_tap("select top 10 * from hsc_v2.hubble_sc2", "test.vot.gz") - INFO: Query finished. [astroquery.utils.tap.core] + >>> result = esahubble.query_tap(query="select top 10 * from hsc.hubble_sc", output_file="test.vot.gz") # doctest: +IGNORE_OUTPUT This will execute an ADQL query to download the first 10 sources in the Hubble Source Catalog (HSC) version 2.1 (format default: compressed @@ -440,6 +452,15 @@ votable). The result of the query will be stored in the file 'test.vot.gz'. The result of this query can be viewed by doing result.get_results() or printing it by doing print(result). +To access the same information shown in eHST Science Archive: +.. doctest-remote-data:: + + >>> from astroquery.esa.hubble import ESAHubble + >>> esahubble = ESAHubble() + >>> result = esahubble.query_tap(query="select top 10 * from ehst.archive", output_file="archive.vot.gz") # doctest: +IGNORE_OUTPUT + + +Deprecation Warning: this method was previously named as query_hst_tap. Please modify your scripts accordingly. ------------------------------------------------------ 9. Getting related members of HAP and HST observations @@ -454,7 +475,7 @@ method returns the simple observations that make it up. >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> result = esahubble.get_member_observations("jdrz0c010") + >>> result = esahubble.get_member_observations(observation_id="jdrz0c010") >>> print(result) ['jdrz0cjxq', 'jdrz0cjyq'] @@ -470,7 +491,7 @@ returns the corresponding HAP or HST observation >>> from astroquery.esa.hubble import ESAHubble >>> esahubble = ESAHubble() - >>> result = esahubble.get_hap_hst_link("hst_16316_71_acs_sbc_f150lp_jec071i9") + >>> result = esahubble.get_hap_hst_link(observation_id="hst_16316_71_acs_sbc_f150lp_jec071i9") >>> print(result) ['jec071i9q'] From f417fc0f73ee9e0258c4a13a413e1b72c60ea9f3 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 11:47:39 +0100 Subject: [PATCH 09/14] eJWST: format improvements --- astroquery/esa/jwst/tests/test_jwsttap.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index be7ede0b9a..a09aeed887 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -895,8 +895,8 @@ def test_query_target_error(self): assert "This target resolver is not allowed" in err.value.args[0] with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target("TEST") - assert "This target name cannot be determined with this resolver: ALL" in err.value.args[0] or "Failed " - "to parse" in err.value.args[0] + assert ("This target name cannot be determined with this resolver: ALL" in err.value.args[0] or + 'Failed to parse' in err.value.args[0]) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="M1", target_resolver="ALL") assert err.value.args[0] in ["This target name cannot be determined " @@ -918,20 +918,20 @@ def test_query_target_error(self): with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="SIMBAD", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: SIMBAD' in err.value.args[0] or "Failed " - "to parse" in err.value.args[0] + assert ('This target name cannot be determined with this resolver: SIMBAD' in err.value.args[0] or + 'Failed to parse' in err.value.args[0]) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="NED", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: NED' in err.value.args[0] or "Failed " - "to parse" in err.value.args[0] + assert ('This target name cannot be determined with this resolver: NED' in err.value.args[0] or\ + 'Failed to parse' in err.value.args[0]) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="VIZIER", radius=units.Quantity(5, units.deg)) - assert 'This target name cannot be determined with this resolver: VIZIER' in err.value.args[0] or "Failed " - "to parse" in err.value.args[0] + assert ('This target name cannot be determined with this resolver: VIZIER' in err.value.args[0] or + 'Failed to parse' in err.value.args[0]) def test_remove_jobs(self): dummyTapHandler = DummyTapHandler() From d126fd99bb4d4ff0bb7c678eb717a25e565f0eb7 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 11:49:58 +0100 Subject: [PATCH 10/14] eHST: pycodestyle issues with unused dependencies --- astroquery/esa/hubble/core.py | 1 - astroquery/esa/hubble/tests/test_esa_hubble.py | 3 +-- astroquery/esa/hubble/tests/test_esa_hubble_remote.py | 1 - astroquery/esa/jwst/tests/test_jwsttap.py | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index d0e9f1570b..2143a17a7e 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -45,7 +45,6 @@ def __init__(self, *, tap_handler=None, show_messages=True): if show_messages: self.get_status_messages() - def download_product(self, observation_id, *, calibration_level=None, filename=None, verbose=False, product_type=None): """ diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index 29e9713ebe..97fc5b7575 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -23,7 +23,6 @@ from astroquery.esa.hubble import ESAHubbleClass from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler -from astroquery.utils.tap.conn.tapconn import ConnectionHandler from astropy.utils.exceptions import AstropyDeprecationWarning @@ -635,5 +634,5 @@ def test_query_hst_tap(self): def test_show_messages(self, mock_conn, mock_res): mock_res.status = 400 mock_conn.getresponse = MagicMock(return_value=mock_res) - ehst = ESAHubbleClass() + ESAHubbleClass() mock_res.assert_called() diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index f3262b084e..99e0f4ac85 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -12,7 +12,6 @@ import tempfile import os -from unittest.mock import patch, MagicMock import pytest from astroquery.esa.hubble import ESAHubble diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index a09aeed887..d97d3cff8a 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -924,7 +924,7 @@ def test_query_target_error(self): with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="test", target_resolver="NED", radius=units.Quantity(5, units.deg)) - assert ('This target name cannot be determined with this resolver: NED' in err.value.args[0] or\ + assert ('This target name cannot be determined with this resolver: NED' in err.value.args[0] or 'Failed to parse' in err.value.args[0]) with pytest.raises((ValueError, TableParseError)) as err: From 11fcb7c66c49b1c71956b2bef97bd1733fe6e5ea Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 17:15:25 +0100 Subject: [PATCH 11/14] eHST: remove warning in deprecated method, tap_client as variable --- astroquery/esa/hubble/core.py | 8 +------- astroquery/utils/tap/core.py | 12 +++++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index 2143a17a7e..e801900996 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -587,9 +587,7 @@ def query_tap(self, query, *, async_job=False, output_file=None, table = job.get_results() return table - @deprecated(since="0.4.7", message=("Use of query_hst_tap method is no longer supported. " - "Please use query_tap method instead, with the same arguments."), - alternative="query_tap") + @deprecated(since="0.4.7", alternative="query_tap") def query_hst_tap(self, query, *, async_job=False, output_file=None, output_format="votable", verbose=False): """Launches a synchronous or asynchronous job to query the HST tap @@ -613,10 +611,6 @@ def query_hst_tap(self, query, *, async_job=False, output_file=None, ------- A table object """ - warnings.warn( - "Use of query_hst_tap method is no longer supported. Please use" - " query_tap method instead, with the same arguments.", - AstropyDeprecationWarning) self.query_tap(query=query, async_job=False, output_file=None, output_format="votable", verbose=False) diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index 87b9a9aabf..6711784903 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -34,7 +34,6 @@ __all__ = ['Tap', 'TapPlus'] VERSION = "20200428.1" -TAP_CLIENT_ID = f"aqtappy1-{VERSION}" class Tap: @@ -138,6 +137,7 @@ def __init__(self, url=None, def __internalInit(self): self.__connHandler = None + self.tap_client_id = f"aqtappy1-{VERSION}" def load_tables(self, verbose=False): """Loads all public tables @@ -585,7 +585,7 @@ def __launchJobMultipart(self, query, uploadResource, uploadTableName, "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": str(outputFormat), - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(self.tap_client_id), "QUERY": str(query), "UPLOAD": "" + str(uploadValue)} if maxrec is not None: @@ -625,7 +625,7 @@ def __launchJob(self, query, outputFormat, context, verbose, name=None, "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": str(outputFormat), - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(self.tap_client_id), "QUERY": str(query)} if maxrec is not None: args['MAXREC'] = maxrec @@ -799,11 +799,9 @@ def __internalInit(self): self.__pwd = None self.__isLoggedIn = False - @staticmethod - def __set_client_id(client_id): + def __set_client_id(self, client_id): if client_id: - global TAP_CLIENT_ID - TAP_CLIENT_ID = client_id + self.tap_client_id = client_id def load_tables(self, only_names=False, include_shared_tables=False, verbose=False): From 3569a03a47a6ac737cb7222417c569f0bceb44aa Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 17:24:29 +0100 Subject: [PATCH 12/14] TAP_CLIENT_ID replaced with tap.tap_client_id --- astroquery/gaia/tests/test_gaiatap.py | 4 ++-- astroquery/utils/tap/tests/test_tap.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/astroquery/gaia/tests/test_gaiatap.py b/astroquery/gaia/tests/test_gaiatap.py index 278d1f92e4..1ec977a4ee 100644 --- a/astroquery/gaia/tests/test_gaiatap.py +++ b/astroquery/gaia/tests/test_gaiatap.py @@ -32,7 +32,7 @@ import numpy as np from astroquery.utils import ASTROPY_LT_4_1 from astroquery.utils.tap.xmlparser import utils -from astroquery.utils.tap.core import TapPlus, TAP_CLIENT_ID +from astroquery.utils.tap.core import TapPlus from astroquery.utils.tap import taputils @@ -89,7 +89,7 @@ def mock_querier_async(): "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", - "tapclient": TAP_CLIENT_ID, + "tapclient": tapplus.tap_client_id, "PHASE": "RUN", "QUERY": ( "SELECT crossmatch_positional('schemaA','tableA','schemaB','tableB',1.0," diff --git a/astroquery/utils/tap/tests/test_tap.py b/astroquery/utils/tap/tests/test_tap.py index 6d5c5458b7..2d0ca01f59 100644 --- a/astroquery/utils/tap/tests/test_tap.py +++ b/astroquery/utils/tap/tests/test_tap.py @@ -23,7 +23,7 @@ from astroquery.utils.tap.conn.tests.DummyConnHandler import DummyConnHandler from astroquery.utils.tap.conn.tests.DummyResponse import DummyResponse -from astroquery.utils.tap.core import TapPlus, TAP_CLIENT_ID +from astroquery.utils.tap.core import TapPlus from astroquery.utils.tap.xmlparser import utils from astroquery.utils.tap import taputils @@ -164,7 +164,7 @@ def test_launch_sync_job(): "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "PHASE": "RUN", "QUERY": str(q)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) @@ -228,7 +228,7 @@ def test_launch_sync_job_redirect(): "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "PHASE": "RUN", "QUERY": str(q)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) @@ -310,7 +310,7 @@ def test_launch_async_job(): "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "PHASE": "RUN", "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) @@ -393,7 +393,7 @@ def test_start_job(): "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable", - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" @@ -451,7 +451,7 @@ def test_abort_job(): "LANG": "ADQL", "FORMAT": "votable", "MAXREC": 10, - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" @@ -485,7 +485,7 @@ def test_job_parameters(): "LANG": "ADQL", "FORMAT": "votable", "MAXREC": 10, - "tapclient": str(TAP_CLIENT_ID), + "tapclient": str(tap.tap_client_id), "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" From 141ce72ce90c77a98fe454b71f7e6bf12bc3d6b1 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Thu, 24 Nov 2022 17:31:51 +0100 Subject: [PATCH 13/14] TAP_CLIENT_ID removed --- docs/utils/tap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utils/tap.rst b/docs/utils/tap.rst index 50ee41c8de..16a54f42ad 100644 --- a/docs/utils/tap.rst +++ b/docs/utils/tap.rst @@ -768,7 +768,7 @@ new columns names. Here is an example: >>> from astroquery.gaia import Gaia, GaiaClass >>> from astroquery.utils.tap.model.tapcolumn import TapColumn - >>> from astroquery.utils.tap.core import TapPlus, TAP_CLIENT_ID + >>> from astroquery.utils.tap.core import TapPlus >>> from astroquery.utils.tap import taputils >>> gaia = GaiaClass(gaia_tap_server='https://gea.esac.esa.int/', gaia_data_server='https://gea.esac.esa.int/') >>> gaia.login() From d5221e587bc2bea7290ce832093a902b64363c1b Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Fri, 25 Nov 2022 15:30:07 +0100 Subject: [PATCH 14/14] HSC version removed in docs --- docs/esa/hubble/hubble.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/esa/hubble/hubble.rst b/docs/esa/hubble/hubble.rst index 6d4d5506f2..e3e2c1ebe9 100644 --- a/docs/esa/hubble/hubble.rst +++ b/docs/esa/hubble/hubble.rst @@ -447,7 +447,7 @@ Access Protocol (TAP) and via the Astronomical Data Query Language (ADQL). >>> result = esahubble.query_tap(query="select top 10 * from hsc.hubble_sc", output_file="test.vot.gz") # doctest: +IGNORE_OUTPUT This will execute an ADQL query to download the first 10 sources in the -Hubble Source Catalog (HSC) version 2.1 (format default: compressed +Hubble Source Catalog (HSC) (format default: compressed votable). The result of the query will be stored in the file 'test.vot.gz'. The result of this query can be viewed by doing result.get_results() or printing it by doing print(result).