diff --git a/Procfile b/Procfile index cfe64a9c..8b0636d5 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,5 @@ -minikube-mount: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-app: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit:/mnt/topcp helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index ec8adf0c..6da3bd25 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -17,9 +17,6 @@ RUN poetry config virtualenvs.create false && \ RUN pip install gunicorn -COPY boot.sh ./ -RUN chmod 755 boot.sh - COPY transformer_capabilities.json ./ RUN chmod 644 transformer_capabilities.json @@ -29,9 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf +RUN chmod 755 /home/servicex/boot.sh + USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["./boot.sh"] +ENTRYPOINT ["/home/servicex/boot.sh"] diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/boot.sh old mode 100644 new mode 100755 index 599ab461..350504fd --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/boot.sh @@ -4,7 +4,7 @@ action=${1:-web_service} if [ "$action" = "web_service" ] ; then mkdir instance - exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" + exec gunicorn -b :5000 --reload --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" fi diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index ac7ff9ac..4923b0d6 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -36,6 +36,11 @@ "ifTrue": ["--no-filter"], "ifFalse": None, }, + "docker_image": { + "properType": str, + "properTypeString": "string", + "optional": True, + }, } @@ -52,16 +57,19 @@ def generate_files_from_query(query, query_file_path): "customConfig", ] - # ensure all keys are specified + # ensure all required keys are specified for key in options: if key not in jquery: + # Skip optional parameters + if options[key].get("optional", False): + continue raise ValueError( key + " must be specified. May be type None or ", options[key]["properTypeString"], ) for key in jquery: - # ensure only aviable options are allowed + # ensure only available options are allowed if key not in options: raise KeyError( key + " is not implemented. Available keys: " + str(options.keys()) @@ -78,7 +86,7 @@ def generate_files_from_query(query, query_file_path): ) # check for reco.yaml, parton.yaml and particle.yaml files - if isinstance(jquery[key], str): + if isinstance(jquery[key], str) and "fileName" in options[key]: with open( os.path.join(query_file_path, options[key]["fileName"]), "w" ) as file: diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 76ebcf0c..8e8a3839 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -62,11 +62,14 @@ def generate_code(self, query, cache_path: str): capabilities_path = os.environ.get( "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) + + # Generate query files first to create any metadata + query_translate.generate_files_from_query(query, query_file_path) + + # Copy capabilities file shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), ) - query_translate.generate_files_from_query(query, query_file_path) - return GeneratedFileResult(_hash, query_file_path) diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 264b1c51..7c2c047b 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -12,21 +12,25 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) # create input.txt file for event loop and insert file_path as only line with open("input.txt", "w") as f: f.write(file_path) - # move reco.yaml, parton.yaml and particle.yaml if they exit to CONFIG_LOC loacation + + # Get CONFIG_LOC with a sensible default (current working directory) + config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) + + # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location if os.path.exists("/generated/reco.yaml"): shutil.copyfile( "/generated/reco.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "reco.yaml"), + os.path.join(config_loc, "reco.yaml"), ) if os.path.exists("/generated/parton.yaml"): shutil.copyfile( "/generated/parton.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "parton.yaml"), + os.path.join(config_loc, "parton.yaml"), ) if os.path.exists("/generated/particle.yaml"): shutil.copyfile( "/generated/particle.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "particle.yaml"), + os.path.join(config_loc, "particle.yaml"), ) generated_transformer.runTop_el() diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index b328e69b..e35ec964 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -183,6 +183,11 @@ data: CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} } {{- end }} + {{- if .Values.codeGen.topcp.enabled }} + # TopCP custom image configuration + TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} + {{- end }} + {{- $didFinders := list }} {{- if .Values.didFinder.CERNOpenData.enabled }} {{- $didFinders = append $didFinders "cernopendata" }} diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 97d225ae..e4b02ffe 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -35,5 +35,18 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 + {{- if eq $codeGenName "topcp" }} + volumeMounts: + - name: host-topcp-volume + mountPath: /home/servicex + {{- end }} + + {{- if eq $codeGenName "topcp" }} + volumes: + - name: host-topcp-volume + hostPath: + path: /mnt/topcp + type: DirectoryOrCreate + {{- end }} {{- end }} {{- end }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 22209e73..cda785e1 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -108,6 +108,9 @@ codeGen: tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 + allowedRepositories: + - "sslhep" + defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: CERNOpenData: @@ -175,7 +178,6 @@ postgresql: image: repository: bitnamilegacy/os-shell tag: 12-debian-12-r51 - global: postgresql: auth: @@ -200,7 +202,6 @@ rabbitmq: image: repository: bitnamilegacy/os-shell tag: 12-debian-12-r51 - extraConfiguration: |- consumer_timeout = 3600000 diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index ea88516c..c088f3c0 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -25,6 +25,8 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from typing import Optional + import requests from requests_toolbelt.multipart import decoder @@ -43,7 +45,11 @@ def post_request(self, post_url, post_obj): return result def generate_code_for_selection( - self, request_record: TransformRequest, namespace: str, user_codegen_name: str + self, + request_record: TransformRequest, + namespace: str, + user_codegen_name: str, + custom_image: Optional[str] = None, ) -> tuple[str, str, str, str]: """ Generates the C++ code for a request's selection string. diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 51e57668..471638b9 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid +import json from datetime import datetime, timezone from typing import Optional, List @@ -99,7 +100,6 @@ def _initialize_dataset_manager( request_id: str, config: dict, ) -> DatasetManager: - # did xor file_list if bool(did) == bool(file_list): raise BadRequest("Must provide did or file-list but not both") @@ -151,7 +151,6 @@ def post(self): uuid.uuid4() ) # make sure we have a request id for all messages try: - try: args = self.parser.parse_args() except BadRequest as bad_request: @@ -212,6 +211,15 @@ def post(self): files=0, ) + print(f"selection: {args.get('selection')}") + custom_docker_image = None + try: + selection = json.loads(args["selection"]) + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + except json.decoder.JSONDecodeError: + pass + # The first thing to do is make sure the requested selection is correct, # and can generate the requested code ( @@ -223,7 +231,10 @@ def post(self): request_rec, namespace, user_codegen_name ) - request_rec.image = codegen_transformer_image + if custom_docker_image: + request_rec.image = custom_docker_image + else: + request_rec.image = codegen_transformer_image # Check to make sure the transformer docker image actually exists (if enabled) if config["TRANSFORMER_VALIDATE_DOCKER_IMAGE"]: