From 03aba7206efa4c2dd75577a6f882acbb0541edb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 18 Mar 2025 19:17:52 -0700 Subject: [PATCH 1/4] Swapping some pyvo calls for astroquery for 1_euclid --- .../1_Euclid_intro_MER_images.md | 47 +++++-------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/tutorials/euclid_access/1_Euclid_intro_MER_images.md b/tutorials/euclid_access/1_Euclid_intro_MER_images.md index 30aeb644..73544f5a 100644 --- a/tutorials/euclid_access/1_Euclid_intro_MER_images.md +++ b/tutorials/euclid_access/1_Euclid_intro_MER_images.md @@ -58,7 +58,7 @@ Each MER image is approximately 1.47 GB. Downloading can take some time. ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed. -# !pip install numpy 'astropy>=5.3' matplotlib pyvo 'sep>=1.4' fsspec pandas +# !pip install numpy 'astropy>=5.3' matplotlib 'astroquery>=0.4.10' 'sep>=1.4' fsspec ``` ```{code-cell} ipython3 @@ -78,7 +78,7 @@ from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStret from astropy.wcs import WCS from astropy import units as u -import pyvo as vo +from astroquery.ipac.irsa import Irsa import sep # Copy-on-write is more performant and avoids unexpected modifications of the original DataFrame. @@ -95,44 +95,23 @@ coord = SkyCoord.from_name('HD 168151') Use IRSA's Simple Image Access (SIA) API to search for all Euclid MER mosaics that overlap with the search region you have specified. We specify the euclid_DpdMerBksMosaic "collection" because it lists all of the multiwavelength MER mosaics, along with their associated catalogs. -```{code-cell} ipython3 -irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA') - -image_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic') -``` - -Convert the table to pandas dataframe - -```{code-cell} ipython3 -df_im_irsa=image_table.to_table().to_pandas().reset_index() -``` +```{tip} +The IRSA SIA collections can be listed using using the ``list_collections`` method, we can filter on the ones containing "euclid" in the collection name: -Change the settings so we can see all the columns in the dataframe and the full column width (to see the full long URL) - -```{code-cell} ipython3 -pd.set_option('display.max_columns', None) -pd.set_option('display.max_colwidth', None) - - -## Can use the following lines to reset the max columns and column width of pandas -# pd.reset_option('display.max_columns') -# pd.reset_option('display.max_colwidth') + Irsa.list_collections(filter='euclid') ``` ```{code-cell} ipython3 -df_im_irsa +image_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic') ``` -This dataframe contains lots of datasets that have been "Euclidized", which means that they have been put on a common pixel scale chosen for the Euclid mission. Choose "science" as the data product subtype to see all science images of this tile. +This table lists all MER mosaic images available in this search position. These mosaics include the Euclid VIS, Y, J, H images, as well as ground-based telescopes which have been put on the same pixel scale. For more information, see the [Euclid documentation at IPAC](https://euclid.caltech.edu/page/euclid-faq-tech/). -```{code-cell} ipython3 -df_im_euclid=df_im_irsa[ (df_im_irsa['dataproduct_subtype']=='science')].reset_index() - -df_im_euclid -``` +Note that there are various image types are returned as well, we filter out the `science` images from these: ```{code-cell} ipython3 -print('There are',len(df_im_euclid),'MER images of this object/MER tile.') +science_images = image_table[image_table['dataproduct_subtype'] == 'science'] +science_images ``` ## 2. Retrieve a Euclid Q1 MER mosaic image in the VIS bandpass @@ -144,8 +123,8 @@ print('There are',len(df_im_euclid),'MER images of this object/MER tile.') Note that 'access_estsize' is in units of kb ```{code-cell} ipython3 -filename = df_im_euclid[df_im_euclid['energy_bandpassname']=='VIS']['access_url'].to_list()[0] -filesize = df_im_euclid[df_im_euclid['energy_bandpassname']=='VIS']['access_estsize'].to_list()[0]/1000000 +filename = science_images[science_images['energy_bandpassname']=='VIS']['access_url'][0] +filesize = science_images[science_images['energy_bandpassname']=='VIS']['access_estsize'][0]/1000000 print(filename) @@ -216,7 +195,7 @@ We'd like to take a look at the multiwavelength images of our object, but the fu ``` ```{code-cell} ipython3 -urls = df_im_euclid['access_url'].to_list() +urls = science_images['access_url'] urls ``` From 16b1f12a5a7ddd4adcab7d16177514c913245bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 18 Mar 2025 19:18:19 -0700 Subject: [PATCH 2/4] Swap some pyvo for astroquery in 2_euclid --- .../2_Euclid_intro_MER_catalog.md | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md b/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md index 3ed5c7bd..718a075e 100644 --- a/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md +++ b/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md @@ -50,27 +50,25 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed -# !pip install numpy matplotlib pyvo +# !pip install numpy matplotlib astroquery>=0.4.10 ``` ```{code-cell} ipython3 import numpy as np import matplotlib.pyplot as plt -import pyvo as vo +from astroquery.ipac.irsa import Irsa ``` ## 1. Download MER catalog from IRSA directly to this notebook -```{code-cell} ipython3 -service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") -``` ++++ + +First, have a look at what Euclid catalogs are available. With the ``list_catalogs`` functionality, we'll receive a list of the name of the catalogs as well as their brief desciption. ```{code-cell} ipython3 -tables = service.tables -for tablename in tables.keys(): - if "tap_schema" not in tablename and "euclid_q1" in tablename: - tables[tablename].describe() +tables = Irsa.list_catalogs(filter='euclid') +tables ``` ### Choose the Euclid MER table @@ -84,13 +82,12 @@ table_mer = 'euclid_q1_mer_catalogue' - List the column names ```{code-cell} ipython3 -columns = tables[table_mer].columns -print(len(columns)) +columns_info = Irsa.list_columns(catalog=table_mer) +print(len(columns_info)) ``` ```{code-cell} ipython3 -for col in columns: - print(f'{f"{col.name}":30s} {col.unit} {col.description}') +columns_info ``` ```{tip} @@ -121,7 +118,7 @@ adql_stars = ("SELECT TOP 10000 mer.object_id, mer.ra, mer.dec, mer.flux_vis_psf # Run the query -result_stars = service.search(adql_stars) +result_stars = Irsa.query_tap(adql_stars) ``` ```{code-cell} ipython3 From 5cd79316658175158c0395cf67816bf62ee34c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 28 Mar 2025 14:21:37 -0700 Subject: [PATCH 3/4] Swapping out pyvo for astroquery in PHZ notebook --- .../4_Euclid_intro_PHZ_catalog.md | 45 ++++++++----------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md b/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md index 8fa3b67a..c63d714c 100644 --- a/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md +++ b/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md @@ -51,7 +51,7 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed. -# !pip install requests matplotlib pandas 'astropy>=5.3' pyvo fsspec firefly_client +# !pip install requests matplotlib pandas 'astropy>=5.3' 'astroquery>=0.4.10' fsspec firefly_client ``` ```{code-cell} ipython3 @@ -72,7 +72,7 @@ from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStret from astropy.wcs import WCS from firefly_client import FireflyClient -import pyvo as vo +from astroquery.ipac.irsa import Irsa ``` ## 1. Find the MER Tile ID that corresponds to a given RA and Dec @@ -92,9 +92,7 @@ coord = SkyCoord(ra, dec, unit='deg', frame='icrs') This searches specifically in the euclid_DpdMerBksMosaic "collection" which is the MER images and catalogs. ```{code-cell} ipython3 -irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA') - -image_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic').to_table() +image_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic') ``` ```{note} @@ -123,40 +121,29 @@ print('The MER tile ID for this object is :',tileID) ## 2. Download PHZ catalog from IRSA -```{code-cell} ipython3 -## Use IRSA to search for catalogs - -service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") +Use IRSA's TAP to search catalogs - -## Search for all tables in IRSA labled as euclid_q1 -tables = service.tables -for tablename in tables.keys(): - if "tap_schema" not in tablename and "euclid_q1" in tablename: - tables[tablename].describe() +```{code-cell} ipython3 +Irsa.list_catalogs(filter='euclid') ``` ```{code-cell} ipython3 -table_mer= 'euclid_q1_mer_catalogue' -table_phz= 'euclid_q1_phz_photo_z' -table_1dspectra= 'euclid.objectid_spectrafile_association_q1' +table_mer = 'euclid_q1_mer_catalogue' +table_phz = 'euclid_q1_phz_photo_z' +table_1dspectra = 'euclid.objectid_spectrafile_association_q1' ``` -### Learn some information about the table: +### Learn some information about the photo-z catalog: - How many columns are there? - List the column names ```{code-cell} ipython3 -columns = tables[table_phz].columns -print(len(columns)) -``` - -```{code-cell} ipython3 -for col in columns: - print(f'{f"{col.name}":30s} {col.unit} {col.description}') ## Currently no descriptions +columns_info = Irsa.list_columns(catalog=table_phz) +print(len(columns_info)) ``` +```{tip} The PHZ catalog contains 67 columns, below are a few highlights: - object_id @@ -164,6 +151,12 @@ The PHZ catalog contains 67 columns, below are a few highlights: - median redshift (phz_median) - phz_classification - phz_90_int1, phz_90_int2 (The phz PDF interval containing 90% of the probability, upper and lower values) +``` + +```{code-cell} ipython3 +# Full list of columns and their description +columns_info +``` ```{note} The phz_catalog on IRSA has more columns than it does on the ESA archive. From 0555f9b90004965491126afe8d96f47c8c14e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sipo=CC=8Bcz?= Date: Fri, 28 Mar 2025 14:33:25 -0700 Subject: [PATCH 4/4] Swapping out more pyvo with astroquery --- .../2_Euclid_intro_MER_catalog.md | 7 ++- .../4_Euclid_intro_PHZ_catalog.md | 6 +-- .../5_Euclid_intro_SPE_catalog.md | 51 ++++++------------- 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md b/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md index 718a075e..604099db 100644 --- a/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md +++ b/tutorials/euclid_access/2_Euclid_intro_MER_catalog.md @@ -50,7 +50,7 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed -# !pip install numpy matplotlib astroquery>=0.4.10 +# !pip install numpy matplotlib 'astroquery>=0.4.10' ``` ```{code-cell} ipython3 @@ -67,8 +67,7 @@ from astroquery.ipac.irsa import Irsa First, have a look at what Euclid catalogs are available. With the ``list_catalogs`` functionality, we'll receive a list of the name of the catalogs as well as their brief desciption. ```{code-cell} ipython3 -tables = Irsa.list_catalogs(filter='euclid') -tables +Irsa.list_catalogs(filter='euclid') ``` ### Choose the Euclid MER table @@ -77,7 +76,7 @@ tables table_mer = 'euclid_q1_mer_catalogue' ``` -### Learn some information about the table: +### Learn some information about the MER catalog: - How many columns are there? - List the column names diff --git a/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md b/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md index c63d714c..f0c2d8f2 100644 --- a/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md +++ b/tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md @@ -221,8 +221,8 @@ AND phz.phz_median BETWEEN 1.4 AND 1.6 \ adql -## Use TAP with this ADQL string using pyvo -result = service.search(adql) +## Use TAP with this ADQL string +result = Irsa.query_tap(adql) ## Convert table to pandas dataframe @@ -319,7 +319,7 @@ FROM {table_1dspectra} \ WHERE objectid = {obj_id}" ## Pull the data on this particular galaxy -result2 = service.search(adql_object) +result2 = Irsa.query_tap(adql_object) df2=result2.to_table().to_pandas() df2 ``` diff --git a/tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md b/tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md index 31e5efb0..fb2b954c 100644 --- a/tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md +++ b/tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md @@ -50,7 +50,7 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed -# !pip install matplotlib pandas astropy pyvo +# !pip install matplotlib pandas astropy 'astroquery>=0.4.10' ``` ```{code-cell} ipython3 @@ -69,7 +69,7 @@ from astropy import units as u from astropy.utils.data import download_file from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStretch -import pyvo as vo +from astroquery.ipac.irsa import Irsa ``` ## 1. Find the MER Tile ID that corresponds to a given RA and Dec @@ -86,12 +86,10 @@ coord = SkyCoord.from_name('HD 168151') This searches specifically in the euclid_DpdMerBksMosaic "collection" which is the MER images and catalogs. ```{code-cell} ipython3 -irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA') - -im_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic') +im_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic') ## Convert the table to pandas dataframe -df_im_irsa=im_table.to_table().to_pandas() +df_im_irsa=im_table.to_pandas() ``` ```{code-cell} ipython3 @@ -125,19 +123,14 @@ print('The MER tile ID for this object is :',tileID) Search for all tables in IRSA labeled as euclid ```{code-cell} ipython3 -service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") - -tables = service.tables -for tablename in tables.keys(): - if "tap_schema" not in tablename and "euclid" in tablename: - tables[tablename].describe() +Irsa.list_catalogs(filter='euclid') ``` ```{code-cell} ipython3 -table_mer= 'euclid_q1_mer_catalogue' -table_galaxy_candidates= 'euclid_q1_spectro_zcatalog_spe_galaxy_candidates' -table_1dspectra= 'euclid.objectid_spectrafile_association_q1' -table_lines= 'euclid_q1_spe_lines_line_features' +table_mer = 'euclid_q1_mer_catalogue' +table_galaxy_candidates = 'euclid_q1_spectro_zcatalog_spe_galaxy_candidates' +table_1dspectra = 'euclid.objectid_spectrafile_association_q1' +table_lines = 'euclid_q1_spe_lines_line_features' ``` ### Learn some information about the table: @@ -145,25 +138,13 @@ table_lines= 'euclid_q1_spe_lines_line_features' - List the column names ```{code-cell} ipython3 -columns = tables[table_lines].columns -print(len(columns)) -``` - -```{code-cell} ipython3 -for col in columns: - print(f'{f"{col.name}":30s} {col.unit} {col.description}') ## Currently no descriptions +columns_info = Irsa.list_columns(catalog=table_lines) +print(len(columns_info)) ``` ```{code-cell} ipython3 -## Change the settings so we can see all the columns in the dataframe and the full column width -## (to see the full long URL) -pd.set_option('display.max_columns', None) -pd.set_option('display.max_colwidth', None) - - -## Can use the following lines to reset the max columns and column width of pandas -# pd.reset_option('display.max_columns') -# pd.reset_option('display.max_colwidth') +# Full list of columns and their description +columns_info ``` ## Find some objects with spectra in our tileID @@ -196,8 +177,8 @@ AND lines.spe_line_flux_gf > 2E-16 \ ORDER BY lines.spe_line_snr_gf DESC \ " -# Use TAP with this ADQL string using pyvo -result = service.search(adql) +# Use TAP with this ADQL string +result = Irsa.query_tap(adql) # Convert table to pandas dataframe and drop duplicates result_table = result.to_qtable() @@ -222,7 +203,7 @@ obj_tab ```{code-cell} ipython3 adql_object = f"SELECT * FROM {table_1dspectra} WHERE objectid = {obj_id}" -result2 = service.search(adql_object) +result2 = Irsa.query_tap(adql_object) df2 = result2.to_table().to_pandas() df2 ```