From e35d9e2d36c8db5c79f7429835224e54aa548f05 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Tue, 7 Jan 2020 12:42:46 -0800 Subject: [PATCH 1/2] Update proto and add custom build steps --- azure_functions_worker/protos/_src/README.md | 2 +- .../protos/_src/src/proto/FunctionRpc.proto | 32 ++++++- .../proto/identity/ClaimsIdentityRpc.proto | 2 +- setup.py | 88 ++++++++++++------- 4 files changed, 85 insertions(+), 39 deletions(-) diff --git a/azure_functions_worker/protos/_src/README.md b/azure_functions_worker/protos/_src/README.md index 0f0f2341f..b22f0bb4b 100644 --- a/azure_functions_worker/protos/_src/README.md +++ b/azure_functions_worker/protos/_src/README.md @@ -1,6 +1,6 @@ # Azure Functions Languge Worker Protobuf -This repository contains the protobuf definition file which defines the gRPC service which is used between the Azure WebJobs Script host and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands. +This repository contains the protobuf definition file which defines the gRPC service which is used between the [Azure Functions Host](https://github.com/Azure/azure-functions-host) and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands. To use this repo in Azure Functions language workers, follow steps below to add this repo as a subtree (*Adding This Repo*). If this repo is already embedded in a language worker repo, follow the steps to update the consumed file (*Pulling Updates*). diff --git a/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto b/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto index 906660bd2..d9d4e4a4e 100644 --- a/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto +++ b/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto @@ -10,8 +10,8 @@ option go_package ="github.com/Azure/azure-functions-go-worker/internal/rpc"; package AzureFunctionsRpcMessages; import "google/protobuf/duration.proto"; -import "azure_functions_worker/protos/ClaimsIdentityRpc.proto"; -import "azure_functions_worker/protos/NullableTypes.proto"; +import "identity/ClaimsIdentityRpc.proto"; +import "shared/NullableTypes.proto"; // Interface exported by the server. service FunctionRpc { @@ -189,6 +189,8 @@ message WorkerStatusResponse { message FunctionEnvironmentReloadRequest { // Environment variables from the current process map environment_variables = 1; + // Current directory of function app + string function_app_directory = 2; } message FunctionEnvironmentReloadResponse { @@ -255,6 +257,21 @@ message InvocationRequest { // binding metadata from trigger map trigger_metadata = 4; + + // Populates activityId, tracestate and tags from host + RpcTraceContext trace_context = 5; +} + +// Host sends ActivityId, traceStateString and Tags from host +message RpcTraceContext { + // This corresponds to Activity.Current?.Id + string trace_parent = 1; + + // This corresponds to Activity.Current?.TraceStateString + string trace_state = 2; + + // This corresponds to Activity.Current?.Tags + map attributes = 3; } // Host requests worker to cancel invocation @@ -368,6 +385,12 @@ message RpcLog { None = 6; } + // Category of the log. Defaults to User if not specified. + enum RpcLogCategory { + User = 0; + System = 1; + } + // Unique id for invocation (if exists) string invocation_id = 1; @@ -389,6 +412,9 @@ message RpcLog { // json serialized property bag, or could use a type scheme like map string properties = 7; + + // Category of the log. Either user(default) or system. + RpcLogCategory log_category = 8; } // Encapsulates an Exception @@ -399,7 +425,7 @@ message RpcException { // Stack trace for the exception string stack_trace = 1; - // Textual message describing hte exception + // Textual message describing the exception string message = 2; } diff --git a/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto b/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto index fb3ee129c..c3945bb8a 100644 --- a/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto +++ b/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto @@ -3,7 +3,7 @@ syntax = "proto3"; option java_package = "com.microsoft.azure.functions.rpc.messages"; -import "azure_functions_worker/protos/NullableTypes.proto"; +import "shared/NullableTypes.proto"; // Light-weight representation of a .NET System.Security.Claims.ClaimsIdentity object. // This is the same serialization as found in EasyAuth, and needs to be kept in sync with diff --git a/setup.py b/setup.py index b4bbff8dd..311b3e4cb 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,8 @@ import tempfile import urllib.request import zipfile +import re +from distutils import dir_util from distutils.command import build from setuptools import setup @@ -73,56 +75,74 @@ def _gen_grpc(self): proto_root_dir = root / 'azure_functions_worker' / 'protos' proto_src_dir = proto_root_dir / '_src' / 'src' / 'proto' - staging_root_dir = root / 'build' / 'protos' + build_dir = root / 'build' + staging_root_dir = build_dir / 'protos' staging_dir = (staging_root_dir / 'azure_functions_worker' / 'protos') - build_dir = staging_dir / 'azure_functions_worker' / 'protos' - built_protos_dir = root / 'build' / 'built_protos' - built_proto_files_dir = (built_protos_dir - / 'azure_functions_worker' / 'protos') + built_protos_dir = build_dir / 'built_protos' - if os.path.exists(build_dir): + if os.path.exists(staging_root_dir): shutil.rmtree(build_dir) if os.path.exists(built_protos_dir): shutil.rmtree(built_protos_dir) - proto_files = glob.glob(str(proto_src_dir / '**' / '*.proto'), - recursive=True) + shutil.copytree(proto_src_dir, staging_dir) - os.makedirs(build_dir) - for proto_file in proto_files: - shutil.copy(proto_file, build_dir) + os.makedirs(built_protos_dir) - protos = [os.path.basename(proto_file) for proto_file in proto_files] + protos = [ + os.sep.join(('shared', 'NullableTypes.proto')), + os.sep.join(('identity', 'ClaimsIdentityRpc.proto')), + 'FunctionRpc.proto' + ] - full_protos = [] for proto in protos: - full_proto = os.sep.join( - ('azure_functions_worker', 'protos', - 'azure_functions_worker', 'protos', proto) - ) - full_protos.append(full_proto) - - os.makedirs(built_protos_dir) - subprocess.run([ - sys.executable, '-m', 'grpc_tools.protoc', - '-I', os.sep.join(('azure_functions_worker', 'protos')), - '--python_out', str(built_protos_dir), - '--grpc_python_out', str(built_protos_dir), - *full_protos - ], check=True, stdout=sys.stdout, stderr=sys.stderr, - cwd=staging_root_dir) - - compiled = glob.glob(str(built_proto_files_dir / '*.py')) - - if not compiled: + subprocess.run([ + sys.executable, '-m', 'grpc_tools.protoc', + '-I', os.sep.join(('azure_functions_worker', 'protos')), + '--python_out', str(built_protos_dir), + '--grpc_python_out', str(built_protos_dir), + os.sep.join(('azure_functions_worker', 'protos', proto)), + ], check=True, stdout=sys.stdout, stderr=sys.stderr, + cwd=staging_root_dir) + + compiled_files = glob.glob( + str(built_protos_dir / '**' / '*.py'), + recursive=True) + + if not compiled_files: print('grpc_tools.protoc produced no Python files', file=sys.stderr) sys.exit(1) - for f in compiled: - shutil.copy(f, proto_root_dir) + # Needed to support absolute imports in files. See + # https://github.com/protocolbuffers/protobuf/issues/1491 + self.make_absolute_imports(compiled_files) + + dir_util.copy_tree(built_protos_dir, str(proto_root_dir)) + + def make_absolute_imports(self, compiled_files): + for compiled in compiled_files: + with open(compiled, 'r+') as f: + content = f.read() + f.seek(0) + # Convert lines of the form: + # import xxx_pb2 as xxx__pb2 to + # from azure_functions_worker.protos import xxx_pb2 as.. + p1 = re.sub( + r'\nimport (.*?_pb2)', + r'\nfrom azure_functions_worker.protos import \g<1>', + content) + # Convert lines of the form: + # from identity import xxx_pb2 as.. to + # from azure_functions_worker.protos.identity import xxx_pb2.. + p2 = re.sub( + r'from ([a-z]*) (import.*_pb2)', + r'from azure_functions_worker.protos.\g<1> \g<2>', + p1) + f.write(p2) + f.truncate() class build(build.build, BuildGRPC): From 2f65672bcdb507d4a1de3a859ec2e56118338b7d Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Tue, 7 Jan 2020 16:14:14 -0800 Subject: [PATCH 2/2] Remove redundant directory deletes --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 311b3e4cb..27da4816d 100644 --- a/setup.py +++ b/setup.py @@ -81,12 +81,9 @@ def _gen_grpc(self): / 'azure_functions_worker' / 'protos') built_protos_dir = build_dir / 'built_protos' - if os.path.exists(staging_root_dir): + if os.path.exists(build_dir): shutil.rmtree(build_dir) - if os.path.exists(built_protos_dir): - shutil.rmtree(built_protos_dir) - shutil.copytree(proto_src_dir, staging_dir) os.makedirs(built_protos_dir)