From 81bcc6326fcb8669f5424fa0adc8ebea96fcefc9 Mon Sep 17 00:00:00 2001 From: Volodymyr Savchenko Date: Mon, 31 May 2021 15:31:12 +0200 Subject: [PATCH 01/14] finished prototype: fetching something --- astroquery/legacysurvey/__init__.py | 41 ++++ astroquery/legacysurvey/core.py | 226 ++++++++++++++++++ astroquery/legacysurvey/tests/__init__.py | 0 .../legacysurvey/tests/setup_package.py | 15 ++ astroquery/legacysurvey/tests/test_module.py | 2 + .../legacysurvey/tests/test_module_remote.py | 23 ++ 6 files changed, 307 insertions(+) create mode 100644 astroquery/legacysurvey/__init__.py create mode 100644 astroquery/legacysurvey/core.py create mode 100644 astroquery/legacysurvey/tests/__init__.py create mode 100644 astroquery/legacysurvey/tests/setup_package.py create mode 100644 astroquery/legacysurvey/tests/test_module.py create mode 100644 astroquery/legacysurvey/tests/test_module_remote.py diff --git a/astroquery/legacysurvey/__init__.py b/astroquery/legacysurvey/__init__.py new file mode 100644 index 0000000000..77d89cadbf --- /dev/null +++ b/astroquery/legacysurvey/__init__.py @@ -0,0 +1,41 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +""" +DESI LegacySurvery + +https://www.legacysurvey.org/ +------------------------- + +:author: Gabriele Barni (Gabriele.Barni@unige.ch) +""" + +# Make the URL of the server, timeout and other items configurable +# See +# for docs and examples on how to do this +# Below is a common use case +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.legacysurvey`. + """ + server = _config.ConfigItem( + ['https://portal.nersc.gov/cfs/cosmo/data/legacysurvey/dr9/', + ], + 'base url') + + timeout = _config.ConfigItem( + 30, + 'Time limit for connecting to template_module server.') + + +conf = Conf() + +# Now import your public class +# Should probably have the same name as your module +from .core import LegacySurvey, LegacySurveyClass + +__all__ = ['LegacySurvey', 'LegacySurveyClass', + 'Conf', 'conf', + ] diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py new file mode 100644 index 0000000000..6683375852 --- /dev/null +++ b/astroquery/legacysurvey/core.py @@ -0,0 +1,226 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + + +# put all imports organized as shown below +# 1. standard library imports + +# 2. third party imports +import astropy.units as u +import astropy.coordinates as coord +import astropy.io.votable as votable +from astropy.table import Table +from astropy.io import fits + +import io + +# 3. local imports - use relative imports +# commonly required local imports shown below as example +# all Query classes should inherit from BaseQuery. +from ..query import BaseQuery +# has common functions required by most modules +from ..utils import commons +# prepend_docstr is a way to copy docstrings between methods +from ..utils import prepend_docstr_nosections +# async_to_sync generates the relevant query tools from _async methods +from ..utils import async_to_sync +# import configurable items declared in __init__.py +from . import conf + + +# export all the public classes and methods +__all__ = ['LegacySurvey', 'LegacySurveyClass'] + +# declare global variables and constants if any + + +# Now begin your main class +# should be decorated with the async_to_sync imported previously +@async_to_sync +class LegacySurveyClass(BaseQuery): + + """ + Not all the methods below are necessary but these cover most of the common + cases, new methods may be added if necessary, follow the guidelines at + + """ + # use the Configuration Items imported from __init__.py to set the URL, + # TIMEOUT, etc. + URL = conf.server + TIMEOUT = conf.timeout + + # all query methods are implemented with an "async" method that handles + # making the actual HTTP request and returns the raw HTTP response, which + # should be parsed by a separate _parse_result method. The query_object + # method is created by async_to_sync automatically. It would look like + # this: + """ + def query_object(object_name, get_query_payload=False) + response = self.query_object_async(object_name, + get_query_payload=get_query_payload) + if get_query_payload: + return response + result = self._parse_result(response, verbose=verbose) + return result + """ + + def query_object_async(self, object_name, get_query_payload=False, + cache=True): + """ + This method is for services that can parse object names. Otherwise + use :meth:`astroquery.template_module.TemplateClass.query_region`. + Put a brief description of what the class does here. + + Parameters + ---------- + object_name : str + name of the identifier to query. + get_query_payload : bool, optional + This should default to False. When set to `True` the method + should return the HTTP request parameters as a dict. + verbose : bool, optional + This should default to `False`, when set to `True` it displays + VOTable warnings. + any_other_param : + similarly list other parameters the method takes + + Returns + ------- + response : `requests.Response` + The HTTP response returned from the service. + All async methods should return the raw HTTP response. + + Examples + -------- + While this section is optional you may put in some examples that + show how to use the method. The examples are written similar to + standard doctests in python. + + """ + # the async method should typically have the following steps: + # 1. First construct the dictionary of the HTTP request params. + # 2. If get_query_payload is `True` then simply return this dict. + # 3. Else make the actual HTTP request and return the corresponding + # HTTP response + # All HTTP requests are made via the `BaseQuery._request` method. This + # use a generic HTTP request method internally, similar to + # `requests.Session.request` of the Python Requests library, but + # with added caching-related tools. + + # See below for an example: + + # first initialize the dictionary of HTTP request parameters + request_payload = dict() + + # Now fill up the dictionary. Here the dictionary key should match + # the exact parameter name as expected by the remote server. The + # corresponding dict value should also be in the same format as + # expected by the server. Additional parsing of the user passed + # value may be required to get it in the right units or format. + # All this parsing may be done in a separate private `_args_to_payload` + # method for cleaner code. + + #request_payload['object_name'] = object_name + # similarly fill up the rest of the dict ... + + if get_query_payload: + return request_payload + # BaseQuery classes come with a _request method that includes a + # built-in caching system + + # TODO: implement here http query as needed + # e.g. I suspect we get files like this one: https://portal.nersc.gov/cfs/cosmo/data/legacysurvey/dr9/north/tractor/000/tractor-0001m002.fits + # to confirm with AG, AN + # if so: + + URL = f"{self.URL}/north/tractor/000/tractor-0001m002.fits" + + response = self._request('GET', URL, params={}, + timeout=self.TIMEOUT, cache=cache) + return response + + # For services that can query coordinates, use the query_region method. + # The pattern is similar to the query_object method. The query_region + # method also has a 'radius' keyword for specifying the radius around + # the coordinates in which to search. If the region is a box, then + # the keywords 'width' and 'height' should be used instead. The coordinates + # may be accepted as an `astropy.coordinates` object or as a string, which + # may be further parsed. + + # similarly we write a query_region_async method that makes the + # actual HTTP request and returns the HTTP response + + def query_region_async(self, coordinates, radius, height, width, + get_query_payload=False, cache=True): + """ + Queries a region around the specified coordinates. + + Parameters + ---------- + coordinates : str or `astropy.coordinates`. + coordinates around which to query + radius : str or `astropy.units.Quantity`. + the radius of the cone search + width : str or `astropy.units.Quantity` + the width for a box region + height : str or `astropy.units.Quantity` + the height for a box region + get_query_payload : bool, optional + Just return the dict of HTTP request parameters. + verbose : bool, optional + Display VOTable warnings or not. + + Returns + ------- + response : `requests.Response` + The HTTP response returned from the service. + All async methods should return the raw HTTP response. + """ + request_payload = self._args_to_payload(coordinates, radius, height, + width) + if get_query_payload: + return request_payload + response = self._request('GET', self.URL, params=request_payload, + timeout=self.TIMEOUT, cache=cache) + return response + + # as we mentioned earlier use various python regular expressions, etc + # to create the dict of HTTP request parameters by parsing the user + # entered values. For cleaner code keep this as a separate private method: + + def _args_to_payload(self, *args, **kwargs): + request_payload = dict() + # code to parse input and construct the dict + # goes here. Then return the dict to the caller + return request_payload + + # the methods above call the private _parse_result method. + # This should parse the raw HTTP response and return it as + # an `astropy.table.Table`. Below is the skeleton: + + def _parse_result(self, response, verbose=False): + # if verbose is False then suppress any VOTable related warnings + if not verbose: + commons.suppress_vo_warnings() + # try to parse the result into an astropy.Table, else + # return the raw result with an informative error message. + try: + # do something with regex to get the result into + # astropy.Table form. return the Table. + data = io.BytesIO(response.content) + table = Table.read(data, hdu=1) + except ValueError: + # catch common errors here, but never use bare excepts + # return raw result/ handle in some way + pass + + return table + + +# the default tool for users to interact with is an instance of the Class +LegacySurvey = LegacySurveyClass() + +# once your class is done, tests should be written +# See ./tests for examples on this + +# Next you should write the docs in astroquery/docs/module_name +# using Sphinx. diff --git a/astroquery/legacysurvey/tests/__init__.py b/astroquery/legacysurvey/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/legacysurvey/tests/setup_package.py b/astroquery/legacysurvey/tests/setup_package.py new file mode 100644 index 0000000000..72b8669485 --- /dev/null +++ b/astroquery/legacysurvey/tests/setup_package.py @@ -0,0 +1,15 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + + +import os + + +# setup paths to the test data +# can specify a single file or a list of files +def get_package_data(): + paths = [os.path.join('data', '*.dat'), + os.path.join('data', '*.xml'), + ] # etc, add other extensions + # you can also enlist files individually by names + # finally construct and return a dict for the sub module + return {'astroquery.template_module.tests': paths} diff --git a/astroquery/legacysurvey/tests/test_module.py b/astroquery/legacysurvey/tests/test_module.py new file mode 100644 index 0000000000..5a4829a079 --- /dev/null +++ b/astroquery/legacysurvey/tests/test_module.py @@ -0,0 +1,2 @@ +""" +""" \ No newline at end of file diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py new file mode 100644 index 0000000000..72e1083969 --- /dev/null +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -0,0 +1,23 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + + +# performs similar tests as test_module.py, but performs +# the actual HTTP request rather than monkeypatching them. +# should be disabled or enabled at will - use the +# remote_data decorator from astropy: + +import pytest +from astropy.table import Table + +@pytest.mark.remote_data +class TestLegacySurveyClass: + # now write tests for each method here + def test_this(self): + import astroquery.legacysurvey + + # TODO: add other parameters + table = astroquery.legacysurvey.LegacySurvey.query_object("Mrk421") # type: Table + + print(table) + + assert len(table) > 10 From bc2d29fb7406ca807e01a614473846147c837bd4 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 3 Jun 2021 12:48:11 +0200 Subject: [PATCH 02/14] test bricks list --- astroquery/legacysurvey/__init__.py | 2 +- astroquery/legacysurvey/core.py | 15 ++++++++ .../legacysurvey/tests/test_module_remote.py | 34 ++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/astroquery/legacysurvey/__init__.py b/astroquery/legacysurvey/__init__.py index 77d89cadbf..a22e8e1ca9 100644 --- a/astroquery/legacysurvey/__init__.py +++ b/astroquery/legacysurvey/__init__.py @@ -21,7 +21,7 @@ class Conf(_config.ConfigNamespace): Configuration parameters for `astroquery.legacysurvey`. """ server = _config.ConfigItem( - ['https://portal.nersc.gov/cfs/cosmo/data/legacysurvey/dr9/', + ['https://portal.nersc.gov/cfs/cosmo/data/legacysurvey/', ], 'base url') diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 6683375852..d10ab6c7d0 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -138,6 +138,21 @@ def query_object_async(self, object_name, get_query_payload=False, timeout=self.TIMEOUT, cache=cache) return response + def query_brick_list_async(self, data_release=9, get_query_payload=False, + cache=True): + """ + + """ + request_payload = dict() + + if get_query_payload: + return request_payload + URL = f"{self.URL}/dr{data_release}/north/survey-bricks-dr{data_release}-north.fits.gz" + + response = self._request('GET', URL, params={}, + timeout=self.TIMEOUT, cache=cache) + return response + # For services that can query coordinates, use the query_region method. # The pattern is similar to the query_object method. The query_region # method also has a 'radius' keyword for specifying the radius around diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index 72e1083969..3add1b65ea 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -12,7 +12,7 @@ @pytest.mark.remote_data class TestLegacySurveyClass: # now write tests for each method here - def test_this(self): + def test_query_object(self): import astroquery.legacysurvey # TODO: add other parameters @@ -21,3 +21,35 @@ def test_this(self): print(table) assert len(table) > 10 + + def test_query_region(self): + import astroquery.legacysurvey + from astropy.coordinates import SkyCoord + from astropy.coordinates import Angle, Latitude, Longitude # Angles + + ra = Angle('0h8m05.63s', unit='hourangle').degree + dec = Angle('+14d50m23.3s', unit='hourangle').degree + radius_input = 3.0 # arcmin + + source = SkyCoord(ra, dec, unit='degree') + radius = Angle(radius_input, unit='arcmin') + + photoobj_fields = ['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'mode', + 'psfFlux_u', 'psfFlux_g', 'psfFlux_r', 'psfFlux_i', 'psfFlux_z', + 'psfFluxIvar_u', 'psfFluxIvar_g', 'psfFluxIvar_r', 'psfFluxIvar_i', 'psfFluxIvar_z', + 'TAI_u', 'TAI_g', 'TAI_r', 'TAI_i', 'TAI_z', 'objID', 'thingId'] + + query1 = astroquery.legacysurvey.LegacySurvey.query_region(source, radius=radius, data_release=16, + photoobj_fields=photoobj_fields) + + print(query1) + + def test_query_brick_list(self): + import astroquery.legacysurvey + + # TODO: add other parameters + table = astroquery.legacysurvey.LegacySurvey.query_brick_list(data_release=9) # type: Table + + print(table) + + assert len(table) > 10 From 187d1ea05cd9323e04acf93e7fbbc81625471d78 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 3 Jun 2021 15:12:15 +0200 Subject: [PATCH 03/14] file written on disk --- astroquery/legacysurvey/core.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index d10ab6c7d0..cd42bb3c3f 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -8,6 +8,7 @@ import astropy.units as u import astropy.coordinates as coord import astropy.io.votable as votable +import requests from astropy.table import Table from astropy.io import fits @@ -148,9 +149,15 @@ def query_brick_list_async(self, data_release=9, get_query_payload=False, if get_query_payload: return request_payload URL = f"{self.URL}/dr{data_release}/north/survey-bricks-dr{data_release}-north.fits.gz" + # TODO make it work with the original request + # response = self._request('GET', URL, params={}, + # timeout=self.TIMEOUT, cache=cache) + + response = requests.get(URL) + + + print("completed fits file request") - response = self._request('GET', URL, params={}, - timeout=self.TIMEOUT, cache=cache) return response # For services that can query coordinates, use the query_region method. @@ -221,8 +228,14 @@ def _parse_result(self, response, verbose=False): try: # do something with regex to get the result into # astropy.Table form. return the Table. - data = io.BytesIO(response.content) - table = Table.read(data, hdu=1) + # data = io.BytesIO(response.content) + # TODO figure out on how to avoid writing in a file + with open('/tmp/file_content', 'wb') as fin: + fin.write(response.content) + + table = Table.read('/tmp/file_content', hdu=1) + + # table = Table.read(data, hdu=1) except ValueError: # catch common errors here, but never use bare excepts # return raw result/ handle in some way From 32af07942ae907e8ed7bda88b3777ba18e533992 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 8 Jun 2021 14:22:18 +0200 Subject: [PATCH 04/14] completed query region --- astroquery/legacysurvey/core.py | 58 +++++++++++++------ .../legacysurvey/tests/test_module_remote.py | 15 ++--- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index cd42bb3c3f..a656e467e7 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -65,7 +65,7 @@ def query_object(object_name, get_query_payload=False) """ def query_object_async(self, object_name, get_query_payload=False, - cache=True): + cache=True, data_release=9): """ This method is for services that can parse object names. Otherwise use :meth:`astroquery.template_module.TemplateClass.query_region`. @@ -133,7 +133,7 @@ def query_object_async(self, object_name, get_query_payload=False, # to confirm with AG, AN # if so: - URL = f"{self.URL}/north/tractor/000/tractor-0001m002.fits" + URL = f"{self.URL}/dr{data_release}/north/tractor/000/tractor-0001m002.fits" response = self._request('GET', URL, params={}, timeout=self.TIMEOUT, cache=cache) @@ -155,7 +155,6 @@ def query_brick_list_async(self, data_release=9, get_query_payload=False, response = requests.get(URL) - print("completed fits file request") return response @@ -171,8 +170,8 @@ def query_brick_list_async(self, data_release=9, get_query_payload=False, # similarly we write a query_region_async method that makes the # actual HTTP request and returns the HTTP response - def query_region_async(self, coordinates, radius, height, width, - get_query_payload=False, cache=True): + def query_region_async(self, coordinates, radius, + get_query_payload=False, cache=True, data_release=9): """ Queries a region around the specified coordinates. @@ -182,10 +181,6 @@ def query_region_async(self, coordinates, radius, height, width, coordinates around which to query radius : str or `astropy.units.Quantity`. the radius of the cone search - width : str or `astropy.units.Quantity` - the width for a box region - height : str or `astropy.units.Quantity` - the height for a box region get_query_payload : bool, optional Just return the dict of HTTP request parameters. verbose : bool, optional @@ -197,13 +192,40 @@ def query_region_async(self, coordinates, radius, height, width, The HTTP response returned from the service. All async methods should return the raw HTTP response. """ - request_payload = self._args_to_payload(coordinates, radius, height, - width) - if get_query_payload: - return request_payload - response = self._request('GET', self.URL, params=request_payload, - timeout=self.TIMEOUT, cache=cache) - return response + # call the brick list + table = self.query_brick_list(data_release=data_release) + # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname + ra1 = table['ra1'] + ra2 = table['ra2'] + dec1 = table['dec1'] + dec2 = table['dec2'] + ra = coordinates.ra.deg + # radius not used for the moment, but it will be in the future + # must find the brick within ra1 and ra2 + dec = coordinates.dec.deg + # must find the brick within dec1 and dec2 + row = None + + for r in table: + ra1 = r['ra1'] + ra2 = r['ra2'] + dec1 = r['dec1'] + dec2 = r['dec2'] + if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: + row = r + break + if row is not None: + brickname = r['brickname'] + raIntPart = "{0:03}".format(int(r['ra1'])) + # to get then the brickname of the line of the table + # extract the integer part of ra1, and in string format (eg 001) + URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" + + response = requests.get(URL) + return response + + else: + return None # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user @@ -213,6 +235,7 @@ def _args_to_payload(self, *args, **kwargs): request_payload = dict() # code to parse input and construct the dict # goes here. Then return the dict to the caller + return request_payload # the methods above call the private _parse_result method. @@ -229,13 +252,14 @@ def _parse_result(self, response, verbose=False): # do something with regex to get the result into # astropy.Table form. return the Table. # data = io.BytesIO(response.content) + # TODO figure out on how to avoid writing in a file with open('/tmp/file_content', 'wb') as fin: fin.write(response.content) table = Table.read('/tmp/file_content', hdu=1) - # table = Table.read(data, hdu=1) + # table = Table.read(data) except ValueError: # catch common errors here, but never use bare excepts # return raw result/ handle in some way diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index 3add1b65ea..b945f33b57 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -9,6 +9,7 @@ import pytest from astropy.table import Table + @pytest.mark.remote_data class TestLegacySurveyClass: # now write tests for each method here @@ -27,20 +28,14 @@ def test_query_region(self): from astropy.coordinates import SkyCoord from astropy.coordinates import Angle, Latitude, Longitude # Angles - ra = Angle('0h8m05.63s', unit='hourangle').degree - dec = Angle('+14d50m23.3s', unit='hourangle').degree + ra = Angle('11h04m27.31s', unit='hourangle').degree + dec = Angle('+38d12m31.8s', unit='hourangle').degree radius_input = 3.0 # arcmin - source = SkyCoord(ra, dec, unit='degree') + coordinates = SkyCoord(ra, dec, unit='degree') radius = Angle(radius_input, unit='arcmin') - photoobj_fields = ['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'mode', - 'psfFlux_u', 'psfFlux_g', 'psfFlux_r', 'psfFlux_i', 'psfFlux_z', - 'psfFluxIvar_u', 'psfFluxIvar_g', 'psfFluxIvar_r', 'psfFluxIvar_i', 'psfFluxIvar_z', - 'TAI_u', 'TAI_g', 'TAI_r', 'TAI_i', 'TAI_z', 'objID', 'thingId'] - - query1 = astroquery.legacysurvey.LegacySurvey.query_region(source, radius=radius, data_release=16, - photoobj_fields=photoobj_fields) + query1 = astroquery.legacysurvey.LegacySurvey.query_region(coordinates=coordinates, radius=radius, data_release=9) print(query1) From c2e0d56e2bc6033237e20f1fd80e8e495d7cfb22 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 9 Jul 2021 16:44:23 +0200 Subject: [PATCH 05/14] returns also southern emishpere --- astroquery/legacysurvey/core.py | 61 +++++++++++++------ .../legacysurvey/tests/test_module_remote.py | 4 +- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index a656e467e7..277801a95b 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -139,7 +139,7 @@ def query_object_async(self, object_name, get_query_payload=False, timeout=self.TIMEOUT, cache=cache) return response - def query_brick_list_async(self, data_release=9, get_query_payload=False, + def query_brick_list_async(self, data_release=9, get_query_payload=False, emisphere="north", cache=True): """ @@ -148,7 +148,7 @@ def query_brick_list_async(self, data_release=9, get_query_payload=False, if get_query_payload: return request_payload - URL = f"{self.URL}/dr{data_release}/north/survey-bricks-dr{data_release}-north.fits.gz" + URL = f"{self.URL}/dr{data_release}/{emisphere}/survey-bricks-dr{data_release}-{emisphere}.fits.gz" # TODO make it work with the original request # response = self._request('GET', URL, params={}, # timeout=self.TIMEOUT, cache=cache) @@ -193,39 +193,64 @@ def query_region_async(self, coordinates, radius, All async methods should return the raw HTTP response. """ # call the brick list - table = self.query_brick_list(data_release=data_release) - # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname - ra1 = table['ra1'] - ra2 = table['ra2'] - dec1 = table['dec1'] - dec2 = table['dec2'] + table_north = self.query_brick_list(data_release=data_release, emisphere="north") + table_south = self.query_brick_list(data_release=data_release, emisphere="south") + # # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname + # ra1 = table_north['ra1'] + # ra2 = table_north['ra2'] + # dec1 = table_north['dec1'] + # dec2 = table_north['dec2'] ra = coordinates.ra.deg # radius not used for the moment, but it will be in the future # must find the brick within ra1 and ra2 dec = coordinates.dec.deg # must find the brick within dec1 and dec2 - row = None + row_north = None + row_south = None - for r in table: + response_north = None + response_south = None + + for r in table_north: + ra1 = r['ra1'] + ra2 = r['ra2'] + dec1 = r['dec1'] + dec2 = r['dec2'] + if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: + row_north = r + break + for r in table_south: ra1 = r['ra1'] ra2 = r['ra2'] dec1 = r['dec1'] dec2 = r['dec2'] if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: - row = r + row_south = r break - if row is not None: - brickname = r['brickname'] - raIntPart = "{0:03}".format(int(r['ra1'])) + + if row_north is not None: + brickname = row_north['brickname'] + raIntPart = "{0:03}".format(int(row_north['ra1'])) + # to get then the brickname of the line of the table # extract the integer part of ra1, and in string format (eg 001) URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - response = requests.get(URL) - return response + response_north = requests.get(URL) + return response_north + + if row_south is not None: + brickname = row_south['brickname'] + raIntPart = "{0:03}".format(int(row_south['ra1'])) + + # to get then the brickname of the line of the table + # extract the integer part of ra1, and in string format (eg 001) + URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" + + response_south = requests.get(URL) + return response_south - else: - return None + return None # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index b945f33b57..1ce3e1b516 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -28,8 +28,8 @@ def test_query_region(self): from astropy.coordinates import SkyCoord from astropy.coordinates import Angle, Latitude, Longitude # Angles - ra = Angle('11h04m27.31s', unit='hourangle').degree - dec = Angle('+38d12m31.8s', unit='hourangle').degree + ra = Angle('11h10m55s', unit='hourangle').degree + dec = Angle('+28d32m37s', unit='hourangle').degree radius_input = 3.0 # arcmin coordinates = SkyCoord(ra, dec, unit='degree') From 07b05b480725f77a90b76407be86467c6b5b5e14 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 19 Aug 2021 09:32:40 +0200 Subject: [PATCH 06/14] including southern emisphere and radius parameter --- astroquery/legacysurvey/core.py | 199 ++++++++++++++---- .../legacysurvey/tests/test_module_remote.py | 6 +- 2 files changed, 156 insertions(+), 49 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 277801a95b..e92232b345 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -8,11 +8,13 @@ import astropy.units as u import astropy.coordinates as coord import astropy.io.votable as votable +from astropy.coordinates import SkyCoord import requests -from astropy.table import Table +from astropy.table import Table, vstack from astropy.io import fits import io +import time # 3. local imports - use relative imports # commonly required local imports shown below as example @@ -195,62 +197,155 @@ def query_region_async(self, coordinates, radius, # call the brick list table_north = self.query_brick_list(data_release=data_release, emisphere="north") table_south = self.query_brick_list(data_release=data_release, emisphere="south") - # # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname - # ra1 = table_north['ra1'] - # ra2 = table_north['ra2'] - # dec1 = table_north['dec1'] - # dec2 = table_north['dec2'] - ra = coordinates.ra.deg + # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname # radius not used for the moment, but it will be in the future # must find the brick within ra1 and ra2 dec = coordinates.dec.deg + ra = coordinates.ra.deg # must find the brick within dec1 and dec2 row_north = None row_south = None - response_north = None - response_south = None - - for r in table_north: - ra1 = r['ra1'] - ra2 = r['ra2'] - dec1 = r['dec1'] - dec2 = r['dec2'] - if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: - row_north = r - break - for r in table_south: - ra1 = r['ra1'] - ra2 = r['ra2'] - dec1 = r['dec1'] - dec2 = r['dec2'] - if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: - row_south = r - break - - if row_north is not None: - brickname = row_north['brickname'] - raIntPart = "{0:03}".format(int(row_north['ra1'])) + row_north_list = [] + row_south_list = [] + + # north table extraction + brick_name = table_north['brickname'] + ra1 = table_north['ra1'] + dec1 = table_north['dec1'] + ra2 = table_north['ra2'] + dec2 = table_north['dec2'] + + corners1 = SkyCoord(ra1, dec1, unit="deg") + corners2 = SkyCoord(ra1, dec2, unit="deg") + corners3 = SkyCoord(ra2, dec1, unit="deg") + corners4 = SkyCoord(ra2, dec2, unit="deg") + + sep1 = coordinates.separation(corners1) + sep2 = coordinates.separation(corners2) + sep3 = coordinates.separation(corners3) + sep4 = coordinates.separation(corners4) + + # for r in table_north: + for i in range(len(table_north)): + if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ + or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): + row_north_list.append(table_north[i]) + # ra1 = r['ra1'] + # ra2 = r['ra2'] + # dec1 = r['dec1'] + # dec2 = r['dec2'] + + # step_deg = (dec2 - dec1) + # numb_steps_deg = step_deg / radius + # + # if numb_steps_deg > 0: + + # skycoord1 = SkyCoord(ra1, dec1, unit='degree') + # skycoord2 = SkyCoord(ra2, dec2, unit='degree') + # + # t0 = time.time() + # sep1 = skycoord1.separation(coordinates) + # print("time for calculating separation1: %s s", time.time() - t0) + # + # t0 = time.time() + # sep2 = skycoord2.separation(coordinates) + # print("time for calculating separation2: %s s", time.time() - t0) + + # if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: + # row_north = r + # break + # if sep1 < radius or sep2 < radius or \ + # (ra1 <= ra <= ra2 and dec1 <= dec <= dec2): + # # query a brick + # row_north_list.append(r) + + # south table extraction + brick_name = table_south['brickname'] + ra1 = table_south['ra1'] + dec1 = table_south['dec1'] + ra2 = table_south['ra2'] + dec2 = table_south['dec2'] + + corners1 = SkyCoord(ra1, dec1, unit="deg") + corners2 = SkyCoord(ra1, dec2, unit="deg") + corners3 = SkyCoord(ra2, dec1, unit="deg") + corners4 = SkyCoord(ra2, dec2, unit="deg") + + sep1 = coordinates.separation(corners1) + sep2 = coordinates.separation(corners2) + sep3 = coordinates.separation(corners3) + sep4 = coordinates.separation(corners4) + + # for r in table_south: + for i in range(len(table_south)): + if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ + or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): + row_south_list.append(table_south[i]) + # ra1 = r['ra1'] + # ra2 = r['ra2'] + # dec1 = r['dec1'] + # dec2 = r['dec2'] + # + # skycoord1 = SkyCoord(ra1, dec1, unit='degree') + # skycoord2 = SkyCoord(ra2, dec2, unit='degree') + # + # sep1 = skycoord1.separation(coordinates) + # sep2 = skycoord2.separation(coordinates) + # + # if sep1 < radius or sep2 < radius: + # # query a brick + # row_south_list.append(r) + + responses = [] + + for r in row_north_list: + brickname = r['brickname'] + raIntPart = "{0:03}".format(int(r['ra1'])) # to get then the brickname of the line of the table # extract the integer part of ra1, and in string format (eg 001) URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - response_north = requests.get(URL) - return response_north + response = requests.get(URL) + if response is not None: + responses.append(response) - if row_south is not None: - brickname = row_south['brickname'] - raIntPart = "{0:03}".format(int(row_south['ra1'])) + for r in row_south_list: + brickname = r['brickname'] + raIntPart = "{0:03}".format(int(r['ra1'])) # to get then the brickname of the line of the table # extract the integer part of ra1, and in string format (eg 001) URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" - response_south = requests.get(URL) - return response_south - - return None + response = requests.get(URL) + if response is not None: + responses.append(response) + + return responses + + # if row_north is not None: + # brickname = row_north['brickname'] + # raIntPart = "{0:03}".format(int(row_north['ra1'])) + # + # # to get then the brickname of the line of the table + # # extract the integer part of ra1, and in string format (eg 001) + # URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" + # + # response_north = requests.get(URL) + # return response_north + # + # if row_south is not None: + # brickname = row_south['brickname'] + # raIntPart = "{0:03}".format(int(row_south['ra1'])) + # + # # to get then the brickname of the line of the table + # # extract the integer part of ra1, and in string format (eg 001) + # URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" + # + # response_south = requests.get(URL) + # return response_south # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user @@ -267,30 +362,42 @@ def _args_to_payload(self, *args, **kwargs): # This should parse the raw HTTP response and return it as # an `astropy.table.Table`. Below is the skeleton: - def _parse_result(self, response, verbose=False): + def _parse_result(self, responses, verbose=False): + tables_list = [] + output_table = Table() + # if verbose is False then suppress any VOTable related warnings if not verbose: commons.suppress_vo_warnings() # try to parse the result into an astropy.Table, else # return the raw result with an informative error message. try: + if not isinstance(responses, list): + responses = [responses] + # do something with regex to get the result into # astropy.Table form. return the Table. # data = io.BytesIO(response.content) + # table = Table.read(data) - # TODO figure out on how to avoid writing in a file - with open('/tmp/file_content', 'wb') as fin: - fin.write(response.content) + for r in responses: + if r.status_code == 200: + # TODO figure out on how to avoid writing in a file + with open('/tmp/file_content', 'wb') as fin: + fin.write(r.content) - table = Table.read('/tmp/file_content', hdu=1) + table = Table.read('/tmp/file_content', hdu=1) + tables_list.append(table) + + if len(tables_list) > 0: + output_table = vstack(tables_list) - # table = Table.read(data) except ValueError: # catch common errors here, but never use bare excepts # return raw result/ handle in some way pass - return table + return output_table # the default tool for users to interact with is an instance of the Class diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index 1ce3e1b516..4f1f9b7eee 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -28,9 +28,9 @@ def test_query_region(self): from astropy.coordinates import SkyCoord from astropy.coordinates import Angle, Latitude, Longitude # Angles - ra = Angle('11h10m55s', unit='hourangle').degree - dec = Angle('+28d32m37s', unit='hourangle').degree - radius_input = 3.0 # arcmin + ra = Angle('11h04m27s', unit='hourangle').degree + dec = Angle('+38d12m32s', unit='hourangle').degree + radius_input = 30.0 # arcmin coordinates = SkyCoord(ra, dec, unit='degree') radius = Angle(radius_input, unit='arcmin') From 2f55a4c14a5ad8a9a5a73da804ed87fb68a143b0 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 19 Aug 2021 09:34:56 +0200 Subject: [PATCH 07/14] removed comments --- astroquery/legacysurvey/core.py | 68 --------------------------------- 1 file changed, 68 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index e92232b345..fa6efedf56 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -202,9 +202,6 @@ def query_region_async(self, coordinates, radius, # must find the brick within ra1 and ra2 dec = coordinates.dec.deg ra = coordinates.ra.deg - # must find the brick within dec1 and dec2 - row_north = None - row_south = None row_north_list = [] row_south_list = [] @@ -231,34 +228,6 @@ def query_region_async(self, coordinates, radius, if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): row_north_list.append(table_north[i]) - # ra1 = r['ra1'] - # ra2 = r['ra2'] - # dec1 = r['dec1'] - # dec2 = r['dec2'] - - # step_deg = (dec2 - dec1) - # numb_steps_deg = step_deg / radius - # - # if numb_steps_deg > 0: - - # skycoord1 = SkyCoord(ra1, dec1, unit='degree') - # skycoord2 = SkyCoord(ra2, dec2, unit='degree') - # - # t0 = time.time() - # sep1 = skycoord1.separation(coordinates) - # print("time for calculating separation1: %s s", time.time() - t0) - # - # t0 = time.time() - # sep2 = skycoord2.separation(coordinates) - # print("time for calculating separation2: %s s", time.time() - t0) - - # if ra1 <= ra <= ra2 and dec1 <= dec <= dec2: - # row_north = r - # break - # if sep1 < radius or sep2 < radius or \ - # (ra1 <= ra <= ra2 and dec1 <= dec <= dec2): - # # query a brick - # row_north_list.append(r) # south table extraction brick_name = table_south['brickname'] @@ -277,25 +246,10 @@ def query_region_async(self, coordinates, radius, sep3 = coordinates.separation(corners3) sep4 = coordinates.separation(corners4) - # for r in table_south: for i in range(len(table_south)): if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): row_south_list.append(table_south[i]) - # ra1 = r['ra1'] - # ra2 = r['ra2'] - # dec1 = r['dec1'] - # dec2 = r['dec2'] - # - # skycoord1 = SkyCoord(ra1, dec1, unit='degree') - # skycoord2 = SkyCoord(ra2, dec2, unit='degree') - # - # sep1 = skycoord1.separation(coordinates) - # sep2 = skycoord2.separation(coordinates) - # - # if sep1 < radius or sep2 < radius: - # # query a brick - # row_south_list.append(r) responses = [] @@ -325,28 +279,6 @@ def query_region_async(self, coordinates, radius, return responses - # if row_north is not None: - # brickname = row_north['brickname'] - # raIntPart = "{0:03}".format(int(row_north['ra1'])) - # - # # to get then the brickname of the line of the table - # # extract the integer part of ra1, and in string format (eg 001) - # URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - # - # response_north = requests.get(URL) - # return response_north - # - # if row_south is not None: - # brickname = row_south['brickname'] - # raIntPart = "{0:03}".format(int(row_south['ra1'])) - # - # # to get then the brickname of the line of the table - # # extract the integer part of ra1, and in string format (eg 001) - # URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" - # - # response_south = requests.get(URL) - # return response_south - # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user # entered values. For cleaner code keep this as a separate private method: From d2bfbda828764f3cd4ce06974d2c8e7fd8d66152 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 19 Aug 2021 11:38:29 +0200 Subject: [PATCH 08/14] merging two loops --- astroquery/legacysurvey/core.py | 88 +++++++++++++------ .../legacysurvey/tests/test_module_remote.py | 5 +- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index fa6efedf56..abf2bbd191 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -206,6 +206,8 @@ def query_region_async(self, coordinates, radius, row_north_list = [] row_south_list = [] + responses = [] + # north table extraction brick_name = table_north['brickname'] ra1 = table_north['ra1'] @@ -223,11 +225,20 @@ def query_region_async(self, coordinates, radius, sep3 = coordinates.separation(corners3) sep4 = coordinates.separation(corners4) - # for r in table_north: + t0 = time.time() + print("Beginning processing bricks northern emishpere") for i in range(len(table_north)): if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): - row_north_list.append(table_north[i]) + # row_north_list.append(table_north[i]) + brickname = brick_name[i] + raIntPart = "{0:03}".format(int(ra1[i])) + URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" + + response = requests.get(URL) + if response is not None and response.status_code == 200: + responses.append(response) + print("Completion processing bricks northern emishpere, total time: ", time.time() - t0) # south table extraction brick_name = table_south['brickname'] @@ -246,36 +257,55 @@ def query_region_async(self, coordinates, radius, sep3 = coordinates.separation(corners3) sep4 = coordinates.separation(corners4) + t0 = time.time() + print("Beginning processing bricks southern emisphere") for i in range(len(table_south)): if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): - row_south_list.append(table_south[i]) - - responses = [] - - for r in row_north_list: - brickname = r['brickname'] - raIntPart = "{0:03}".format(int(r['ra1'])) - - # to get then the brickname of the line of the table - # extract the integer part of ra1, and in string format (eg 001) - URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - - response = requests.get(URL) - if response is not None: - responses.append(response) - - for r in row_south_list: - brickname = r['brickname'] - raIntPart = "{0:03}".format(int(r['ra1'])) - - # to get then the brickname of the line of the table - # extract the integer part of ra1, and in string format (eg 001) - URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" - - response = requests.get(URL) - if response is not None: - responses.append(response) + # row_south_list.append(table_south[i]) + brickname = brick_name[i] + raIntPart = "{0:03}".format(int(ra1[i])) + URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" + + response = requests.get(URL) + if response is not None and response.status_code == 200: + responses.append(response) + print("Completion processing bricks southern emisphere, total time: ", time.time() - t0) + print("-----------------------------------------------------") + + + # t0 = time.time() + # print("Beginning requesting northern tractors, row_north_list size: ", len(row_north_list)) + # for r in row_north_list: + # brickname = r['brickname'] + # raIntPart = "{0:03}".format(int(r['ra1'])) + # + # # to get then the brickname of the line of the table + # # extract the integer part of ra1, and in string format (eg 001) + # URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" + # + # response = requests.get(URL) + # if response is not None and response.status_code == 200: + # responses.append(response) + # print("Completion requests northern tractors, total time: ", time.time() - t0) + # + # + # t0 = time.time() + # print("Beginning processing requesting southern tractors, row_south_list size: ", len(row_south_list)) + # for r in row_south_list: + # brickname = r['brickname'] + # raIntPart = "{0:03}".format(int(r['ra1'])) + # # brickname = r[0] + # # raIntPart = "{0:03}".format(int(r[1])) + # + # # to get then the brickname of the line of the table + # # extract the integer part of ra1, and in string format (eg 001) + # URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" + # + # response = requests.get(URL) + # if response is not None and response.status_code == 200: + # responses.append(response) + # print("Completion requests southern tractors, total time: ", time.time() - t0) return responses diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index 4f1f9b7eee..741c390ad2 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -5,6 +5,7 @@ # the actual HTTP request rather than monkeypatching them. # should be disabled or enabled at will - use the # remote_data decorator from astropy: +import time import pytest from astropy.table import Table @@ -24,6 +25,8 @@ def test_query_object(self): assert len(table) > 10 def test_query_region(self): + t0 = time.time() + print("Beginning test") import astroquery.legacysurvey from astropy.coordinates import SkyCoord from astropy.coordinates import Angle, Latitude, Longitude # Angles @@ -36,7 +39,7 @@ def test_query_region(self): radius = Angle(radius_input, unit='arcmin') query1 = astroquery.legacysurvey.LegacySurvey.query_region(coordinates=coordinates, radius=radius, data_release=9) - + print("Test completion: ", time.time() - t0) print(query1) def test_query_brick_list(self): From 27adbe52447135baa0475e5e11241bd4e1fa8fa5 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 24 Aug 2021 18:18:11 +0200 Subject: [PATCH 09/14] removed useless comments --- astroquery/legacysurvey/core.py | 36 --------------------------------- 1 file changed, 36 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index abf2bbd191..1b7d5d3507 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -203,8 +203,6 @@ def query_region_async(self, coordinates, radius, dec = coordinates.dec.deg ra = coordinates.ra.deg - row_north_list = [] - row_south_list = [] responses = [] @@ -273,40 +271,6 @@ def query_region_async(self, coordinates, radius, print("Completion processing bricks southern emisphere, total time: ", time.time() - t0) print("-----------------------------------------------------") - - # t0 = time.time() - # print("Beginning requesting northern tractors, row_north_list size: ", len(row_north_list)) - # for r in row_north_list: - # brickname = r['brickname'] - # raIntPart = "{0:03}".format(int(r['ra1'])) - # - # # to get then the brickname of the line of the table - # # extract the integer part of ra1, and in string format (eg 001) - # URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - # - # response = requests.get(URL) - # if response is not None and response.status_code == 200: - # responses.append(response) - # print("Completion requests northern tractors, total time: ", time.time() - t0) - # - # - # t0 = time.time() - # print("Beginning processing requesting southern tractors, row_south_list size: ", len(row_south_list)) - # for r in row_south_list: - # brickname = r['brickname'] - # raIntPart = "{0:03}".format(int(r['ra1'])) - # # brickname = r[0] - # # raIntPart = "{0:03}".format(int(r[1])) - # - # # to get then the brickname of the line of the table - # # extract the integer part of ra1, and in string format (eg 001) - # URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" - # - # response = requests.get(URL) - # if response is not None and response.status_code == 200: - # responses.append(response) - # print("Completion requests southern tractors, total time: ", time.time() - t0) - return responses # as we mentioned earlier use various python regular expressions, etc From 2068feae92cee0c292aebece1952eea4f3ce759e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Oct 2021 15:41:37 +0200 Subject: [PATCH 10/14] tap query region --- astroquery/legacysurvey/core.py | 103 ++++-------------- .../legacysurvey/tests/test_module_remote.py | 2 +- 2 files changed, 25 insertions(+), 80 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 1b7d5d3507..2af1b57bec 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -29,7 +29,8 @@ # import configurable items declared in __init__.py from . import conf - +import pyvo as vo +from numpy import pi, cos # export all the public classes and methods __all__ = ['LegacySurvey', 'LegacySurveyClass'] @@ -194,84 +195,25 @@ def query_region_async(self, coordinates, radius, The HTTP response returned from the service. All async methods should return the raw HTTP response. """ - # call the brick list - table_north = self.query_brick_list(data_release=data_release, emisphere="north") - table_south = self.query_brick_list(data_release=data_release, emisphere="south") - # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname - # radius not used for the moment, but it will be in the future - # must find the brick within ra1 and ra2 - dec = coordinates.dec.deg - ra = coordinates.ra.deg - - - responses = [] - - # north table extraction - brick_name = table_north['brickname'] - ra1 = table_north['ra1'] - dec1 = table_north['dec1'] - ra2 = table_north['ra2'] - dec2 = table_north['dec2'] - - corners1 = SkyCoord(ra1, dec1, unit="deg") - corners2 = SkyCoord(ra1, dec2, unit="deg") - corners3 = SkyCoord(ra2, dec1, unit="deg") - corners4 = SkyCoord(ra2, dec2, unit="deg") - - sep1 = coordinates.separation(corners1) - sep2 = coordinates.separation(corners2) - sep3 = coordinates.separation(corners3) - sep4 = coordinates.separation(corners4) - - t0 = time.time() - print("Beginning processing bricks northern emishpere") - for i in range(len(table_north)): - if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ - or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): - # row_north_list.append(table_north[i]) - brickname = brick_name[i] - raIntPart = "{0:03}".format(int(ra1[i])) - URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" - - response = requests.get(URL) - if response is not None and response.status_code == 200: - responses.append(response) - print("Completion processing bricks northern emishpere, total time: ", time.time() - t0) - - # south table extraction - brick_name = table_south['brickname'] - ra1 = table_south['ra1'] - dec1 = table_south['dec1'] - ra2 = table_south['ra2'] - dec2 = table_south['dec2'] - - corners1 = SkyCoord(ra1, dec1, unit="deg") - corners2 = SkyCoord(ra1, dec2, unit="deg") - corners3 = SkyCoord(ra2, dec1, unit="deg") - corners4 = SkyCoord(ra2, dec2, unit="deg") - - sep1 = coordinates.separation(corners1) - sep2 = coordinates.separation(corners2) - sep3 = coordinates.separation(corners3) - sep4 = coordinates.separation(corners4) - - t0 = time.time() - print("Beginning processing bricks southern emisphere") - for i in range(len(table_south)): - if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ - or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): - # row_south_list.append(table_south[i]) - brickname = brick_name[i] - raIntPart = "{0:03}".format(int(ra1[i])) - URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" - - response = requests.get(URL) - if response is not None and response.status_code == 200: - responses.append(response) - print("Completion processing bricks southern emisphere, total time: ", time.time() - t0) - print("-----------------------------------------------------") - - return responses + + # TAP query + # Download tractor catalogue + url = 'https://datalab.noirlab.edu/tap' + tap_service = vo.dal.TAPService(url) + qstr = "SELECT all * FROM ls_dr" + str(data_release) + ".tractor WHERE dec>" + str(coordinates.dec.deg - radius.deg) + " and dec<" + str( + coordinates.dec.deg + radius.deg) + " and ra>" + str(coordinates.ra.deg - radius.deg / cos(coordinates.dec.deg * pi / 180.)) + " and ra<" + str( + coordinates.ra.deg + radius.deg / cos(coordinates.dec.deg * pi / 180)) + print(qstr) + + tap_result = tap_service.run_sync(qstr) + tap_result = tap_result.to_table() + # ra = tap_result['ra'] + # dec = tap_result['dec'] + mask = tap_result['type'] != 'D' + masked_table = tap_result[mask] + + return masked_table + # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user @@ -292,6 +234,9 @@ def _parse_result(self, responses, verbose=False): tables_list = [] output_table = Table() + if isinstance(responses, Table): + return responses + # if verbose is False then suppress any VOTable related warnings if not verbose: commons.suppress_vo_warnings() diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index 741c390ad2..b10059d63f 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -33,7 +33,7 @@ def test_query_region(self): ra = Angle('11h04m27s', unit='hourangle').degree dec = Angle('+38d12m32s', unit='hourangle').degree - radius_input = 30.0 # arcmin + radius_input = 30 # arcmin coordinates = SkyCoord(ra, dec, unit='degree') radius = Angle(radius_input, unit='arcmin') From 87e3c0b3606618488e50193b71ede79d89e2f4da Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Oct 2021 16:21:07 +0200 Subject: [PATCH 11/14] option to still not use tap, default uses it --- astroquery/legacysurvey/core.py | 114 +++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 18 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 2af1b57bec..10d43b14c8 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -174,7 +174,7 @@ def query_brick_list_async(self, data_release=9, get_query_payload=False, emisph # actual HTTP request and returns the HTTP response def query_region_async(self, coordinates, radius, - get_query_payload=False, cache=True, data_release=9): + get_query_payload=False, cache=True, data_release=9, use_tap=True): """ Queries a region around the specified coordinates. @@ -196,23 +196,101 @@ def query_region_async(self, coordinates, radius, All async methods should return the raw HTTP response. """ - # TAP query - # Download tractor catalogue - url = 'https://datalab.noirlab.edu/tap' - tap_service = vo.dal.TAPService(url) - qstr = "SELECT all * FROM ls_dr" + str(data_release) + ".tractor WHERE dec>" + str(coordinates.dec.deg - radius.deg) + " and dec<" + str( - coordinates.dec.deg + radius.deg) + " and ra>" + str(coordinates.ra.deg - radius.deg / cos(coordinates.dec.deg * pi / 180.)) + " and ra<" + str( - coordinates.ra.deg + radius.deg / cos(coordinates.dec.deg * pi / 180)) - print(qstr) - - tap_result = tap_service.run_sync(qstr) - tap_result = tap_result.to_table() - # ra = tap_result['ra'] - # dec = tap_result['dec'] - mask = tap_result['type'] != 'D' - masked_table = tap_result[mask] - - return masked_table + if use_tap: + # TAP query` + # Download tractor catalogue + url = 'https://datalab.noirlab.edu/tap' + tap_service = vo.dal.TAPService(url) + qstr = "SELECT all * FROM ls_dr" + str(data_release) + ".tractor WHERE dec>" + str(coordinates.dec.deg - radius.deg) + " and dec<" + str( + coordinates.dec.deg + radius.deg) + " and ra>" + str(coordinates.ra.deg - radius.deg / cos(coordinates.dec.deg * pi / 180.)) + " and ra<" + str( + coordinates.ra.deg + radius.deg / cos(coordinates.dec.deg * pi / 180)) + print(qstr) + + tap_result = tap_service.run_sync(qstr) + tap_result = tap_result.to_table() + # filter out duplicated lines from the table + mask = tap_result['type'] != 'D' + filtered_table = tap_result[mask] + + return filtered_table + else: + # call the brick list + table_north = self.query_brick_list(data_release=data_release, emisphere="north") + table_south = self.query_brick_list(data_release=data_release, emisphere="south") + # needed columns: ra1, ra2, dec1, dec2 (corners of the bricks), and also brickname + # radius not used for the moment, but it will be in the future + # must find the brick within ra1 and ra2 + dec = coordinates.dec.deg + ra = coordinates.ra.deg + + responses = [] + + # north table extraction + brick_name = table_north['brickname'] + ra1 = table_north['ra1'] + dec1 = table_north['dec1'] + ra2 = table_north['ra2'] + dec2 = table_north['dec2'] + + corners1 = SkyCoord(ra1, dec1, unit="deg") + corners2 = SkyCoord(ra1, dec2, unit="deg") + corners3 = SkyCoord(ra2, dec1, unit="deg") + corners4 = SkyCoord(ra2, dec2, unit="deg") + + sep1 = coordinates.separation(corners1) + sep2 = coordinates.separation(corners2) + sep3 = coordinates.separation(corners3) + sep4 = coordinates.separation(corners4) + + t0 = time.time() + print("Beginning processing bricks northern emishpere") + for i in range(len(table_north)): + if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ + or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): + # row_north_list.append(table_north[i]) + brickname = brick_name[i] + raIntPart = "{0:03}".format(int(ra1[i])) + URL = f"{self.URL}/dr{data_release}/north/tractor/{raIntPart}/tractor-{brickname}.fits" + + response = requests.get(URL) + if response is not None and response.status_code == 200: + responses.append(response) + print("Completion processing bricks northern emishpere, total time: ", time.time() - t0) + + # south table extraction + brick_name = table_south['brickname'] + ra1 = table_south['ra1'] + dec1 = table_south['dec1'] + ra2 = table_south['ra2'] + dec2 = table_south['dec2'] + + corners1 = SkyCoord(ra1, dec1, unit="deg") + corners2 = SkyCoord(ra1, dec2, unit="deg") + corners3 = SkyCoord(ra2, dec1, unit="deg") + corners4 = SkyCoord(ra2, dec2, unit="deg") + + sep1 = coordinates.separation(corners1) + sep2 = coordinates.separation(corners2) + sep3 = coordinates.separation(corners3) + sep4 = coordinates.separation(corners4) + + t0 = time.time() + print("Beginning processing bricks southern emisphere") + for i in range(len(table_south)): + if ((ra1[i] < ra < ra2[i]) and (dec1[i] < dec < dec2[i])) \ + or (sep1[i] < radius) or (sep2[i] < radius) or (sep3[i] < radius) or (sep4[i] < radius): + # row_south_list.append(table_south[i]) + brickname = brick_name[i] + raIntPart = "{0:03}".format(int(ra1[i])) + URL = f"{self.URL}/dr{data_release}/south/tractor/{raIntPart}/tractor-{brickname}.fits" + + response = requests.get(URL) + if response is not None and response.status_code == 200: + responses.append(response) + print("Completion processing bricks southern emisphere, total time: ", time.time() - t0) + print("-----------------------------------------------------") + + return responses # as we mentioned earlier use various python regular expressions, etc From 51e1fa789b6dde9112d9a75328d6922c4173bd6e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Oct 2021 16:06:40 +0200 Subject: [PATCH 12/14] first get_images implementation LegacySurvey --- astroquery/legacysurvey/core.py | 26 +++++++++++++++++++ .../legacysurvey/tests/test_module_remote.py | 21 ++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 10d43b14c8..c482d17fe5 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -39,6 +39,9 @@ # Now begin your main class # should be decorated with the async_to_sync imported previously +from ..utils.commons import FileContainer + + @async_to_sync class LegacySurveyClass(BaseQuery): @@ -292,6 +295,26 @@ def query_region_async(self, coordinates, radius, return responses + def get_images_async(self, coordinates=None, data_release=9, + projection=None, pixels=None, scaling=None, + sampler=None, resolver=None, deedger=None, lut=None, + grid=None, gridlabels=None, radius=None, height=None, + width=None, cache=True, show_progress=True, image_band='g'): + """ + Returns + ------- + A list of context-managers that yield readable file-like objects + """ + + image_size_arcsec = radius.arcsec + pixsize = 2 * image_size_arcsec / pixels + + image_url = 'https://www.legacysurvey.org/viewer/fits-cutout?ra=' + str(coordinates.ra.deg) + '&dec=' + str(coordinates.dec.deg) + '&size=' + str( + pixels) + '&layer=ls-dr' + str(data_release) + '&pixscale=' + str(pixsize) + '&bands=' + image_band + + print("image_url: ", image_url) + + return commons.FileContainer(image_url, encoding='binary', show_progress=show_progress) # as we mentioned earlier use various python regular expressions, etc # to create the dict of HTTP request parameters by parsing the user @@ -315,6 +338,9 @@ def _parse_result(self, responses, verbose=False): if isinstance(responses, Table): return responses + if isinstance(responses, FileContainer): + return responses + # if verbose is False then suppress any VOTable related warnings if not verbose: commons.suppress_vo_warnings() diff --git a/astroquery/legacysurvey/tests/test_module_remote.py b/astroquery/legacysurvey/tests/test_module_remote.py index b10059d63f..b39e27f3a0 100644 --- a/astroquery/legacysurvey/tests/test_module_remote.py +++ b/astroquery/legacysurvey/tests/test_module_remote.py @@ -33,7 +33,7 @@ def test_query_region(self): ra = Angle('11h04m27s', unit='hourangle').degree dec = Angle('+38d12m32s', unit='hourangle').degree - radius_input = 30 # arcmin + radius_input = 5 # arcmin coordinates = SkyCoord(ra, dec, unit='degree') radius = Angle(radius_input, unit='arcmin') @@ -42,6 +42,25 @@ def test_query_region(self): print("Test completion: ", time.time() - t0) print(query1) + def test_get_images(self): + t0 = time.time() + print("Beginning test") + import astroquery.legacysurvey + from astropy.coordinates import SkyCoord + from astropy.coordinates import Angle, Latitude, Longitude # Angles + + ra = Angle('11h04m27s', unit='hourangle').degree + dec = Angle('+38d12m32s', unit='hourangle').degree + radius_input = 1 # arcmin + + coordinates = SkyCoord(ra, dec, unit='degree') + radius = Angle(radius_input, unit='arcmin') + + query1 = astroquery.legacysurvey.LegacySurvey.get_images(coordinates=coordinates, radius=radius, data_release=9, + pixels=60) + print("Test completion: ", time.time() - t0) + print(query1) + def test_query_brick_list(self): import astroquery.legacysurvey From 7e9e13feb95a345eabf1cc6d41636ebf124cd45b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 1 Nov 2021 17:55:44 +0100 Subject: [PATCH 13/14] consistent signature --- astroquery/legacysurvey/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index c482d17fe5..5ec1027cb1 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -200,14 +200,13 @@ def query_region_async(self, coordinates, radius, """ if use_tap: - # TAP query` + # TAP query # Download tractor catalogue url = 'https://datalab.noirlab.edu/tap' tap_service = vo.dal.TAPService(url) qstr = "SELECT all * FROM ls_dr" + str(data_release) + ".tractor WHERE dec>" + str(coordinates.dec.deg - radius.deg) + " and dec<" + str( coordinates.dec.deg + radius.deg) + " and ra>" + str(coordinates.ra.deg - radius.deg / cos(coordinates.dec.deg * pi / 180.)) + " and ra<" + str( coordinates.ra.deg + radius.deg / cos(coordinates.dec.deg * pi / 180)) - print(qstr) tap_result = tap_service.run_sync(qstr) tap_result = tap_result.to_table() @@ -295,7 +294,7 @@ def query_region_async(self, coordinates, radius, return responses - def get_images_async(self, coordinates=None, data_release=9, + def get_images_async(self, position, survey, coordinates=None, data_release=9, projection=None, pixels=None, scaling=None, sampler=None, resolver=None, deedger=None, lut=None, grid=None, gridlabels=None, radius=None, height=None, From 3d88b538c7e5000f4784e1ea4271d2bbf35d98b9 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 15 Nov 2021 11:36:31 +0100 Subject: [PATCH 14/14] use position instead of coordinates --- astroquery/legacysurvey/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/legacysurvey/core.py b/astroquery/legacysurvey/core.py index 5ec1027cb1..6c568cff57 100644 --- a/astroquery/legacysurvey/core.py +++ b/astroquery/legacysurvey/core.py @@ -308,7 +308,7 @@ def get_images_async(self, position, survey, coordinates=None, data_release=9, image_size_arcsec = radius.arcsec pixsize = 2 * image_size_arcsec / pixels - image_url = 'https://www.legacysurvey.org/viewer/fits-cutout?ra=' + str(coordinates.ra.deg) + '&dec=' + str(coordinates.dec.deg) + '&size=' + str( + image_url = 'https://www.legacysurvey.org/viewer/fits-cutout?ra=' + str(position.ra.deg) + '&dec=' + str(position.dec.deg) + '&size=' + str( pixels) + '&layer=ls-dr' + str(data_release) + '&pixscale=' + str(pixsize) + '&bands=' + image_band print("image_url: ", image_url)