From 529e9c5c626ab156682778d6c7c5448841ac76c6 Mon Sep 17 00:00:00 2001 From: KyungEon Choi Date: Wed, 24 Jan 2024 14:14:22 -0600 Subject: [PATCH 1/2] Update examples for servicex backend v1.3.2 --- examples/bigger_uproot.py | 2 +- examples/config_databinder.yaml | 8 +++---- examples/databinder.py | 1 - examples/databinder_python_script.py | 31 ++++++++++++++++++++++++++++ examples/dataset_group.py | 2 +- examples/python_codegen.py | 12 +++++------ examples/single_file_uproot.py | 2 +- examples/xAODDataset.py | 4 ++-- 8 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 examples/databinder_python_script.py diff --git a/examples/bigger_uproot.py b/examples/bigger_uproot.py index 859a91c1..3dd8e34c 100644 --- a/examples/bigger_uproot.py +++ b/examples/bigger_uproot.py @@ -30,7 +30,7 @@ from servicex import ResultFormat from servicex import ServiceXClient -sx = ServiceXClient(backend="production") +sx = ServiceXClient(backend="uc-af") dataset_id = RucioDatasetIdentifier("user.kchoi:user.kchoi.fcnc_tHq_ML.ttH.v8") diff --git a/examples/config_databinder.yaml b/examples/config_databinder.yaml index 9799b498..0c18b27e 100644 --- a/examples/config_databinder.yaml +++ b/examples/config_databinder.yaml @@ -1,8 +1,8 @@ General: - ServiceX: servicex-uc-af + ServiceX: uc-af Codegen: python OutputFormat: root - Delivery: ObjectStore + Delivery: LocalCache Sample: - Name: ttH @@ -25,7 +25,7 @@ Definition: def run_query(input_filenames=None): import uproot - with uproot.open(f"{input_filenames}:nominal") as o: + with uproot.open({input_filenames:"nominal"}) as o: br = o.arrays("mu_pt") return br @@ -33,6 +33,6 @@ Definition: def run_query(input_filenames=None): import uproot - with uproot.open(f"{input_filenames}:mini") as o: + with uproot.open({input_filenames:"mini"}) as o: br = o.arrays("lep_pt") return br \ No newline at end of file diff --git a/examples/databinder.py b/examples/databinder.py index 559f0ed5..eb7840b9 100644 --- a/examples/databinder.py +++ b/examples/databinder.py @@ -28,7 +28,6 @@ from servicex.databinder import DataBinder - sx = DataBinder("config_databinder.yaml") out_dict = sx.deliver() diff --git a/examples/databinder_python_script.py b/examples/databinder_python_script.py new file mode 100644 index 00000000..c69a7922 --- /dev/null +++ b/examples/databinder_python_script.py @@ -0,0 +1,31 @@ +from servicex.databinder import DataBinder + + +def run_query(input_filenames=None): + import uproot + with uproot.open({input_filenames:"mini"}) as o: + br = o.arrays("lep_pt") + return br + + +config = { + "General": + { + "ServiceX": "uc-af", + "Codegen": "python", + "Delivery": "LocalCache" + }, + "Sample": + [ + { + "Name": "ttH", + "XRootDFiles": "root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root", + "Function": run_query + } + ] +} + +sx = DataBinder(config=config) +o = sx.deliver() + +print(o) diff --git a/examples/dataset_group.py b/examples/dataset_group.py index 5fee7fc1..5e78adb3 100644 --- a/examples/dataset_group.py +++ b/examples/dataset_group.py @@ -4,7 +4,7 @@ dataset_id = RucioDatasetIdentifier("user.kchoi:user.kchoi.fcnc_tHq_ML.ttH.v8", num_files=3) -sx = ServiceXClient(backend="production") +sx = ServiceXClient(backend="uc-af") ds_raw = sx.func_adl_dataset( dataset_id, codegen="uproot", diff --git a/examples/python_codegen.py b/examples/python_codegen.py index 5c50b698..5db3e62f 100644 --- a/examples/python_codegen.py +++ b/examples/python_codegen.py @@ -28,18 +28,18 @@ from servicex import FileListDataset from servicex import ServiceXClient +from servicex import ResultFormat -sx = ServiceXClient(backend="testing4") +sx = ServiceXClient(backend="uc-af") dataset_id = FileListDataset("root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root") # NOQA 501 -ds = sx.python_dataset(dataset_id, codegen="python", title="Python") - +ds = sx.python_dataset(dataset_id, codegen="python", title="Python", result_format=ResultFormat.parquet) def run_query(input_filenames=None): import uproot - o = uproot.lazy({input_filenames: "mini"}) - return o.lep_pt - + with uproot.open({input_filenames:"mini"}) as o: + br = o.arrays("lep_pt") + return br sx3 = ds.with_uproot_function(run_query).as_pandas() print(sx3) diff --git a/examples/single_file_uproot.py b/examples/single_file_uproot.py index ac11ccfc..522bab23 100644 --- a/examples/single_file_uproot.py +++ b/examples/single_file_uproot.py @@ -30,7 +30,7 @@ from servicex import ResultFormat from servicex import ServiceXClient -sx = ServiceXClient(backend="testing4") +sx = ServiceXClient(backend="uc-af") dataset_id = FileListDataset("root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root") # NOQA 501 ds = sx.func_adl_dataset(dataset_id, codegen="uproot", diff --git a/examples/xAODDataset.py b/examples/xAODDataset.py index 58e6ca92..4044ff59 100644 --- a/examples/xAODDataset.py +++ b/examples/xAODDataset.py @@ -31,10 +31,10 @@ # A Z to ee sample - Release 21 ds_name = ( - r"mc16_13TeV: mc16_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee" + r"mc16_13TeV:mc16_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee" r".deriv.DAOD_PHYS.e3601_e5984_s3126_r10201_r10210_p5313") -sx = ServiceXClient(backend="testing4") +sx = ServiceXClient(backend="uc-af") did = RucioDatasetIdentifier(ds_name, num_files=10) ds_raw = sx.func_adl_dataset( From 48f2f2ebcd82cc648a34a260959f864b856cffbf Mon Sep 17 00:00:00 2001 From: KyungEon Choi Date: Wed, 24 Jan 2024 14:25:08 -0600 Subject: [PATCH 2/2] Ignore flake8 for examples with python ftn --- examples/databinder_python_script.py | 3 ++- examples/python_codegen.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/databinder_python_script.py b/examples/databinder_python_script.py index c69a7922..578a6ee2 100644 --- a/examples/databinder_python_script.py +++ b/examples/databinder_python_script.py @@ -1,9 +1,10 @@ +# flake8: noqa from servicex.databinder import DataBinder def run_query(input_filenames=None): import uproot - with uproot.open({input_filenames:"mini"}) as o: + with uproot.open({input_filenames:"mini"}) as o: br = o.arrays("lep_pt") return br diff --git a/examples/python_codegen.py b/examples/python_codegen.py index 5db3e62f..62cfbb9e 100644 --- a/examples/python_codegen.py +++ b/examples/python_codegen.py @@ -1,3 +1,4 @@ +# flake8: noqa # Copyright (c) 2022, IRIS-HEP # All rights reserved. # @@ -33,13 +34,20 @@ sx = ServiceXClient(backend="uc-af") dataset_id = FileListDataset("root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root") # NOQA 501 -ds = sx.python_dataset(dataset_id, codegen="python", title="Python", result_format=ResultFormat.parquet) +ds = sx.python_dataset( + dataset_id, + codegen="python", + title="Python", + result_format=ResultFormat.parquet +) + def run_query(input_filenames=None): import uproot - with uproot.open({input_filenames:"mini"}) as o: + with uproot.open({input_filenames:"mini"}) as o: br = o.arrays("lep_pt") return br + sx3 = ds.with_uproot_function(run_query).as_pandas() print(sx3)