From a5865bf163f91a563bbe6d992a5e53f5b9e0cfd5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 2 Jul 2025 07:27:22 +0530 Subject: [PATCH 1/8] api,server,schema,plugin,client: logs web server This feature enables administrators to view management server logs directly in the UI through a dedicated API call. It leverages a Netty-based websocket server to stream logs in real time, offering an efficient way to monitor and debug server operations. Note that the plugin is disabled by default and must be enabled manually. Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 10 +- .../apache/cloudstack/api/BaseResponse.java | 14 + .../apache/cloudstack/api/ResponseObject.java | 3 + .../user/config/ListCapabilitiesCmd.java | 1 + .../api/response/CapabilitiesResponse.java | 9 + client/pom.xml | 5 + .../util/StringListJsonConverter.java | 50 +++ .../db/schema-42010to42100-cleanup.sql | 2 + .../META-INF/db/schema-42010to42100.sql | 17 + plugins/logs-web-server/pom.xml | 41 ++ .../cloudstack/logsws/LogsWebSession.java | 37 ++ .../logsws/LogsWebSessionApiService.java | 35 ++ .../logsws/LogsWebSessionApiServiceImpl.java | 174 ++++++++ .../logsws/LogsWebSessionManager.java | 80 ++++ .../logsws/LogsWebSessionManagerImpl.java | 281 +++++++++++++ .../logsws/LogsWebSessionWebSocket.java | 46 ++ .../admin/CreateLogsWebSessionCmd.java | 101 +++++ .../command/admin/DeleteLogsWebSession.java | 85 ++++ .../command/admin/ListLogsWebSessionsCmd.java | 70 ++++ .../api/response/LogsWebSessionResponse.java | 135 ++++++ .../LogsWebSessionWebSocketResponse.java | 86 ++++ .../logsws/dao/LogsWebSessionDao.java | 33 ++ .../logsws/dao/LogsWebSessionDaoImpl.java | 101 +++++ .../logreader/FilteredLogTailerListener.java | 71 ++++ .../server/LogsWebSocketBroadcastHandler.java | 175 ++++++++ .../server/LogsWebSocketRouteManager.java | 51 +++ .../server/LogsWebSocketRoutingHandler.java | 107 +++++ .../logsws/server/LogsWebSocketServer.java | 123 ++++++ .../server/LogsWebSocketServerHelper.java | 28 ++ .../logsws/vo/LogsWebSessionVO.java | 192 +++++++++ .../logs-web-server/module.properties | 18 + .../spring-logs-web-server-context.xml | 33 ++ .../logsws/LogsWebSessionManagerImplTest.java | 52 +++ plugins/pom.xml | 2 + pom.xml | 6 + .../main/java/com/cloud/api/ApiServer.java | 39 +- .../main/java/com/cloud/api/ApiServlet.java | 11 +- .../cloud/server/ManagementServerImpl.java | 4 +- tools/apidoc/gen_toc.py | 4 +- ui/public/locales/en.json | 2 + ui/src/components/page/GlobalLayout.vue | 23 +- ui/src/components/view/LogsConsole.vue | 396 ++++++++++++++++++ ui/src/utils/plugins.js | 55 ++- ui/src/views/AutogenView.vue | 11 +- ui/src/views/compute/DeployVM.vue | 3 +- 45 files changed, 2787 insertions(+), 35 deletions(-) create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/util/StringListJsonConverter.java create mode 100644 plugins/logs-web-server/pom.xml create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSession.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiService.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiServiceImpl.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManager.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImpl.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionWebSocket.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/CreateLogsWebSessionCmd.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/DeleteLogsWebSession.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/ListLogsWebSessionsCmd.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionResponse.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionWebSocketResponse.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDao.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDaoImpl.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/logreader/FilteredLogTailerListener.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketBroadcastHandler.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRouteManager.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRoutingHandler.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServer.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServerHelper.java create mode 100644 plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/vo/LogsWebSessionVO.java create mode 100644 plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/module.properties create mode 100644 plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/spring-logs-web-server-context.xml create mode 100644 plugins/logs-web-server/src/test/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImplTest.java create mode 100644 ui/src/components/view/LogsConsole.vue diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 8fca652518f2..4058ef225ec1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -98,8 +98,8 @@ public class ApiConstants { public static final String CONVERT_INSTANCE_HOST_ID = "convertinstancehostid"; public static final String CONVERT_INSTANCE_STORAGE_POOL_ID = "convertinstancepoolid"; public static final String ENABLED_REVOCATION_CHECK = "enabledrevocationcheck"; - public static final String CLIENT_ADDRESS = "clientaddress"; public static final String COMBINED_CAPACITY_ORDERING = "COMBINED"; + public static final String CONTEXT_ID = "contextid"; public static final String CONTROLLER = "controller"; public static final String CONTROLLER_UNIT = "controllerunit"; public static final String CONSOLE_ENDPOINT_CREATOR_ADDRESS = "consoleendpointcreatoraddress"; @@ -124,6 +124,7 @@ public class ApiConstants { public static final String DEST_CIDR_LIST = "destcidrlist"; public static final String CLEANUP = "cleanup"; public static final String MAKEREDUNDANT = "makeredundant"; + public static final String CLIENT_ADDRESS = "clientaddress"; public static final String CLUSTER_ID = "clusterid"; public static final String CLUSTER_IDS = "clusterids"; public static final String CLUSTER_NAME = "clustername"; @@ -137,12 +138,14 @@ public class ApiConstants { public static final String CNI_CONFIG_NAME = "cniconfigname"; public static final String CSI_ENABLED = "csienabled"; public static final String COMPONENT = "component"; + public static final String CONNECTED = "connected"; public static final String CPU = "CPU"; public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket"; public static final String CPU_NUMBER = "cpunumber"; public static final String CPU_SPEED = "cpuspeed"; public static final String CPU_LOAD_AVERAGE = "cpuloadaverage"; public static final String CREATED = "created"; + public static final String CREATOR_ADDRESS = "creatoraddress"; public static final String CROSS_ZONE_INSTANCE_CREATION = "crosszoneinstancecreation"; public static final String CTX_ACCOUNT_ID = "ctxaccountid"; public static final String CTX_DETAILS = "ctxDetails"; @@ -161,6 +164,7 @@ public class ApiConstants { public static final String ENCRYPT_ROOT = "encryptroot"; public static final String ENCRYPTION_SUPPORTED = "encryptionsupported"; public static final String ETCD_IPS = "etcdips"; + public static final String FILTERS = "filters"; public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; @@ -1312,6 +1316,8 @@ public class ApiConstants { public static final String WEBHOOK_ID = "webhookid"; public static final String WEBHOOK_NAME = "webhookname"; + public static final String WEBSOCKET = "websocket"; + public static final String NFS_MOUNT_OPTIONS = "nfsmountopts"; public static final String MOUNT_OPTIONS = "mountopts"; @@ -1328,6 +1334,8 @@ public class ApiConstants { public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String LOGS_WEB_SERVER_ENABLED = "logswebserverenabled"; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseResponse.java b/api/src/main/java/org/apache/cloudstack/api/BaseResponse.java index 45016c1a2a26..7c235787dbc1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseResponse.java @@ -32,6 +32,10 @@ public abstract class BaseResponse implements ResponseObject { @Param(description = "the current status of the latest async job acting on this object") private Integer jobStatus; + @SerializedName(ApiConstants.CONTEXT_ID) + @Param(description = "the ID of the executing context") + private String contextId; + public BaseResponse() { } @@ -83,4 +87,14 @@ public Integer getJobStatus() { public void setJobStatus(Integer jobStatus) { this.jobStatus = jobStatus; } + + @Override + public String getContextId() { + return contextId; + } + + @Override + public void setContextId(String contextId) { + this.contextId = contextId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseObject.java b/api/src/main/java/org/apache/cloudstack/api/ResponseObject.java index ff2e172b70b3..1029d3529522 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseObject.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseObject.java @@ -76,6 +76,9 @@ public interface ResponseObject { */ void setJobStatus(Integer jobStatus); + String getContextId(); + void setContextId(String contextId); + public enum ResponseView { Full, Restricted diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index ed1bd7b063b2..599ec4d8411a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -76,6 +76,7 @@ public void execute() { response.setExtensionsPath((String)capabilities.get(ApiConstants.EXTENSIONS_PATH)); response.setDynamicScalingEnabled((Boolean) capabilities.get(ApiConstants.DYNAMIC_SCALING_ENABLED)); response.setAdditionalConfigEnabled((Boolean) capabilities.get(ApiConstants.ADDITONAL_CONFIG_ENABLED)); + response.setLogsWebServerEnabled((Boolean)capabilities.get(ApiConstants.LOGS_WEB_SERVER_ENABLED)); response.setObjectName("capability"); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index d2c71b5f3525..87c2a160c9eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -153,6 +153,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "true if additional configurations or extraconfig can be passed to Instances", since = "4.20.2") private Boolean additionalConfigEnabled; + @SerializedName(ApiConstants.LOGS_WEB_SERVER_ENABLED) + @Param(description = "true if Logs Web Server plugin is enabled, false otherwise", since = "4.21.0") + private boolean logsWebServerEnabled; + public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -279,5 +283,10 @@ public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { public void setAdditionalConfigEnabled(Boolean additionalConfigEnabled) { this.additionalConfigEnabled = additionalConfigEnabled; + + } + + public void setLogsWebServerEnabled(boolean logsWebServerEnabled) { + this.logsWebServerEnabled = logsWebServerEnabled; } } diff --git a/client/pom.xml b/client/pom.xml index d8fa433d5be3..5a6e23f08634 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -246,6 +246,11 @@ cloud-plugin-user-two-factor-authenticator-staticpin ${project.version} + + org.apache.cloudstack + cloud-plugin-logs-web-server + ${project.version} + org.apache.cloudstack cloud-plugin-metrics diff --git a/engine/schema/src/main/java/org/apache/cloudstack/util/StringListJsonConverter.java b/engine/schema/src/main/java/org/apache/cloudstack/util/StringListJsonConverter.java new file mode 100644 index 000000000000..47bc872fa7ef --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/util/StringListJsonConverter.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import java.io.IOException; +import java.util.List; + +@Converter +public class StringListJsonConverter implements AttributeConverter, String> { + + private static final ObjectMapper mapper = new ObjectMapper(); + + @Override + public String convertToDatabaseColumn(List attribute) { + try { + return attribute == null ? null : mapper.writeValueAsString(attribute); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Error converting list to JSON", e); + } + } + + @Override + public List convertToEntityAttribute(String dbData) { + try { + return dbData == null ? null : mapper.readValue(dbData, List.class); + } catch (IOException e) { + throw new IllegalArgumentException("Error converting JSON to list", e); + } + } +} + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql index 5f257f2965bd..b28f85147bde 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql @@ -18,3 +18,5 @@ --; -- Schema upgrade cleanup from 4.20.1.0 to 4.21.0.0 --; + +DROP TABLE IF EXISTS `cloud`.`logs_web_session`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 167dd92730cc..68b938896354 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -208,6 +208,7 @@ SET `sort_key` = CASE END; -- End: Changes for Guest OS category cleanup +<<<<<<< HEAD -- Update description for configuration: host.capacityType.to.order.clusters UPDATE `cloud`.`configuration` SET `description` = 'The host capacity type (CPU, RAM or COMBINED) is used by deployment planner to order clusters during VM resource allocation' @@ -757,3 +758,19 @@ SET `cs`.`domain_id` = ( -- Re-apply VPC: update default network offering for vpc tier to conserve_mode=1 (#8309) UPDATE `cloud`.`network_offerings` SET conserve_mode = 1 WHERE name = 'DefaultIsolatedNetworkOfferingForVpcNetworks'; + +-- Create table for logs web session +CREATE TABLE IF NOT EXISTS `cloud`.`logs_web_session` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + `uuid` varchar(40) NOT NULL COMMENT 'UUID generated for the session', + `filter` varchar(64) DEFAULT NULL COMMENT 'Filter keyword for the session', + `created` datetime NOT NULL COMMENT 'When the session was created', + `domain_id` bigint(20) unsigned NOT NULL COMMENT 'Domain of the account who generated the session', + `account_id` bigint(20) unsigned NOT NULL COMMENT 'Account who generated the session', + `creator_address` VARCHAR(45) DEFAULT NULL COMMENT 'Address of the creator of the session', + `connections` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of connections for the session', + `connected_time` datetime DEFAULT NULL COMMENT 'When the session was connected', + `client_address` VARCHAR(45) DEFAULT NULL COMMENT 'Address of the client that connected to the session', + `removed` datetime COMMENT 'When the session was removed/used', + CONSTRAINT `uc_logs_web_session__uuid` UNIQUE (`uuid`) +); diff --git a/plugins/logs-web-server/pom.xml b/plugins/logs-web-server/pom.xml new file mode 100644 index 000000000000..d01fecd9d387 --- /dev/null +++ b/plugins/logs-web-server/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + cloud-plugin-logs-web-server + Apache CloudStack Plugin - Logs Web Server + + org.apache.cloudstack + cloudstack-plugins + 4.21.0.0-SNAPSHOT + ../pom.xml + + + + org.apache.cloudstack + cloud-api + ${project.version} + + + io.netty + netty-all + + + diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSession.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSession.java new file mode 100644 index 000000000000..c77ab28800c5 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSession.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface LogsWebSession extends ControlledEntity, Identity, InternalIdentity { + long getId(); + List getFilters(); + long getDomainId(); + long getAccountId(); + int getConnections(); + Date getConnectedTime(); + String getCreatorAddress(); + String getClientAddress(); + Date getCreated(); +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiService.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiService.java new file mode 100644 index 000000000000..85e8f2893a7d --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiService.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.logsws.api.command.admin.CreateLogsWebSessionCmd; +import org.apache.cloudstack.logsws.api.command.admin.DeleteLogsWebSession; +import org.apache.cloudstack.logsws.api.command.admin.ListLogsWebSessionsCmd; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionResponse; + +import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; + +public interface LogsWebSessionApiService extends PluggableService { + + ListResponse listLogsWebSessions(ListLogsWebSessionsCmd cmd); + LogsWebSessionResponse createLogsWebSession(CreateLogsWebSessionCmd cmd) throws CloudRuntimeException; + boolean deleteLogsWebSession(DeleteLogsWebSession cmd) throws CloudRuntimeException; + LogsWebSessionResponse createLogsWebSessionResponse(long logsEndpointId); +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiServiceImpl.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiServiceImpl.java new file mode 100644 index 000000000000..eb9d1529d401 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionApiServiceImpl.java @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.logsws.api.command.admin.CreateLogsWebSessionCmd; +import org.apache.cloudstack.logsws.api.command.admin.DeleteLogsWebSession; +import org.apache.cloudstack.logsws.api.command.admin.ListLogsWebSessionsCmd; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionResponse; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionWebSocketResponse; +import org.apache.cloudstack.logsws.dao.LogsWebSessionDao; +import org.apache.cloudstack.logsws.vo.LogsWebSessionVO; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.api.ApiServlet; +import com.cloud.domain.Domain; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.DomainService; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; + +public class LogsWebSessionApiServiceImpl implements LogsWebSessionApiService { + + @Inject + LogsWebSessionManager logsWSManager; + @Inject + LogsWebSessionDao logsWebSessionDao; + @Inject + AccountService accountService; + @Inject + DomainService domainService; + + @Override + public ListResponse listLogsWebSessions(ListLogsWebSessionsCmd cmd) { + final Long id = cmd.getId(); + List responsesList = new ArrayList<>(); + SearchBuilder sb = logsWebSessionDao.createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + SearchCriteria sc = sb.create(); + if (id != null) { + sc.setParameters("id", id); + } + + Filter searchFilter = new Filter(LogsWebSessionVO.class, "id", true, cmd.getStartIndex(), + cmd.getPageSizeVal()); + Pair, Integer> webhooksAndCount = logsWebSessionDao.searchAndCount(sc, searchFilter); + for (LogsWebSessionVO webhook : webhooksAndCount.first()) { + LogsWebSessionResponse response = createLogsWebSessionResponse(webhook); + responsesList.add(response); + } + ListResponse response = new ListResponse<>(); + response.setResponses(responsesList, webhooksAndCount.second()); + return response; + } + + @Override + public LogsWebSessionResponse createLogsWebSession(CreateLogsWebSessionCmd cmd) throws CloudRuntimeException { + final List filters = cmd.getFilters(); + final String extraSecurityToken = cmd.getExtraSecurityToken(); + String clientAddress = null; + Map params = cmd.getFullUrlParams(); + if (MapUtils.isNotEmpty(params)) { + clientAddress = params.get(ApiServlet.CLIENT_INET_ADDRESS_KEY); + } + for (String filter : filters) { + if (StringUtils.isBlank(filter)) { + throw new InvalidParameterValueException(String.format("Invalid value for parameter - %s", + ApiConstants.FILTERS)); + } + } + if (!logsWSManager.canCreateNewLogsWebSession()) { + throw new CloudRuntimeException("Max Logs Web Session limit reached"); + } + final Account account = CallContext.current().getCallingAccount(); + LogsWebSessionVO logsWebSessionVO = new LogsWebSessionVO(filters, account.getDomainId(), account.getAccountId(), + clientAddress); + logsWebSessionVO = logsWebSessionDao.persist(logsWebSessionVO); + return createLogsWebSessionResponse(logsWebSessionVO); + } + + @Override + public boolean deleteLogsWebSession(DeleteLogsWebSession cmd) throws CloudRuntimeException { + final long id = cmd.getId(); + return logsWebSessionDao.remove(id); + } + + protected Set getLogsWebSessionWebSocketResponses( + final LogsWebSessionVO logsWebSessionVO) { + Set responses = new HashSet<>(); + List webSockets = logsWSManager.getLogsWebSessionWebSockets(logsWebSessionVO); + for (LogsWebSessionWebSocket socket : webSockets) { + LogsWebSessionWebSocketResponse webSocketResponse = new LogsWebSessionWebSocketResponse(); + webSocketResponse.setManagementServerId(socket.getManagementServerHost().getUuid()); + webSocketResponse.setManagementServerName(socket.getManagementServerHost().getName()); + webSocketResponse.setHost(socket.getManagementServerHost().getServiceIP()); + webSocketResponse.setPort(socket.getPort()); + webSocketResponse.setPath(socket.getPath()); + responses.add(webSocketResponse); + } + return responses; + } + + protected LogsWebSessionResponse createLogsWebSessionResponse(final LogsWebSessionVO logsWebSessionVO) { + LogsWebSessionResponse response = new LogsWebSessionResponse(); + response.setObjectName("logswebsession"); + response.setId(logsWebSessionVO.getUuid()); + response.setFilters(logsWebSessionVO.getFilters()); + Account account = accountService.getAccount(logsWebSessionVO.getAccountId()); + response.setAccountName(account.getAccountName()); + Domain domain = domainService.getDomain(logsWebSessionVO.getDomainId()); + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); + response.setDomainPath(domain.getName()); + response.setCreatorAddress(logsWebSessionVO.getCreatorAddress()); + response.setConnected(logsWebSessionVO.getConnections()); + response.setClientAddress(logsWebSessionVO.getClientAddress()); + response.setCreated(logsWebSessionVO.getCreated()); + response.setWebsocketResponse(getLogsWebSessionWebSocketResponses(logsWebSessionVO)); + return response; + } + + @Override + public LogsWebSessionResponse createLogsWebSessionResponse(long logsEndpointId) { + LogsWebSessionVO logsWebSessionVO = logsWebSessionDao.findById(logsEndpointId); + if (logsWebSessionVO == null) { + return null; + } + return createLogsWebSessionResponse(logsWebSessionVO); + } + + @Override + public List> getCommands() { + if (!LogsWebSessionManager.LogsWebServerEnabled.value()) { + return Collections.emptyList(); + } + return List.of( + CreateLogsWebSessionCmd.class, + ListLogsWebSessionsCmd.class, + DeleteLogsWebSession.class + ); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManager.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManager.java new file mode 100644 index 000000000000..9150ce866472 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManager.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import java.util.List; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.utils.component.PluggableService; + +public interface LogsWebSessionManager extends PluggableService, Configurable { + int WS_PORT = 8822; + String WS_PATH = "/logger"; + + ConfigKey LogsWebServerEnabled = new ConfigKey<>("Advanced", Boolean.class, + "logs.web.server.enabled", "false", + "Indicates whether Logs Web Server plugin is enabled or not", + false); + + ConfigKey LogsWebServerConcurrentSessions = new ConfigKey<>("Advanced", Integer.class, + "logs.web.server.concurrent.sessions", "1", + "Number of concurrent sessions that can be created at a time. To allow unlimited a value of zero can used", + true); + + ConfigKey LogsWebServerSessionStaleCleanupInterval = new ConfigKey<>("Advanced", Integer.class, + "logs.web.server.session.stale.cleanup.interval", "600", + "Time(in seconds) after which a stale (not connected or disconnected) Logs Web Server session will be automatically deleted", + false); + + ConfigKey LogsWebServerPort = new ConfigKey<>("Advanced", Integer.class, + "logs.web.server.port", String.valueOf(WS_PORT), + "The port to be used for Logs Web Server", + false, + ConfigKey.Scope.ManagementServer); + + ConfigKey LogsWebServerPath = new ConfigKey<>("Advanced", String.class, + "logs.web.server.path", WS_PATH, + "The path prefix to be used for Logs Web Server", + false, + ConfigKey.Scope.ManagementServer); + + ConfigKey LogsWebServerSessionIdleTimeout = new ConfigKey<>("Advanced", Integer.class, + "logs.web.server.session.idle.timeout", "60", + "Time(in seconds) after which a Logs Web Server session will be automatically disconnected if in idle state", + false, + ConfigKey.Scope.ManagementServer); + + ConfigKey LogsWebServerLogFile = new ConfigKey<>("Advanced", String.class, + "logs.web.server.log.file", "/var/logs/cloudstack/management/management-server.log", + "Log file to be used by Logs Web Server", + true, + ConfigKey.Scope.ManagementServer); + + ConfigKey LogsWebServerSessionTailExistingLines = new ConfigKey<>("Advanced", Integer.class, + "logs.web.server.session.tail.existing.lines", "512", + "Number of existing lines to be read from the logs file on session connect", + true, + ConfigKey.Scope.ManagementServer); + + void startWebSocketServer(); + void stopWebSocketServer(); + List getLogsWebSessionWebSockets(final LogsWebSession logsWebSession); + boolean canCreateNewLogsWebSession(); +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImpl.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImpl.java new file mode 100644 index 000000000000..804216130e2e --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImpl.java @@ -0,0 +1,281 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.logsws.dao.LogsWebSessionDao; +import org.apache.cloudstack.logsws.server.LogsWebSocketServer; +import org.apache.cloudstack.logsws.server.LogsWebSocketServerHelper; +import org.apache.cloudstack.logsws.vo.LogsWebSessionVO; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.utils.DateUtil; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; + +public class LogsWebSessionManagerImpl extends ManagerBase implements LogsWebSessionManager, LogsWebSocketServerHelper { + + @Inject + LogsWebSessionDao logsWebSessionDao; + @Inject + ManagementServerHostDao managementServerHostDao; + + private int serverPort; + private String serverPath; + private int serverIdleTimeoutSeconds; + private LogsWebSocketServer loggerWebSocketServer; + private ScheduledExecutorService staleLogsWebSessionCleanupExecutor; + private Long managementServerId = null; + + protected Long getManagementServerId() { + if (managementServerId != null) { + ManagementServerHostVO managementServerHostVO = + managementServerHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (managementServerHostVO != null) { + managementServerId = managementServerHostVO.getId(); + } + } + return managementServerId; + } + + @Override + public String getConfigComponentName() { + return LogsWebSessionManager.class.getSimpleName(); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + staleLogsWebSessionCleanupExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Logs-Web-Sessions-Stale-Cleanup-Worker")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure " + LogsWebSessionManagerImpl.class.getSimpleName()); + } + return true; + } + + @Override + public boolean start() { + if (!LogsWebServerEnabled.value()) { + return true; + } + serverPort = LogsWebServerPort.valueIn(getManagementServerId()); + serverPath = LogsWebServerPath.valueIn(getManagementServerId()); + serverIdleTimeoutSeconds = LogsWebServerSessionIdleTimeout.valueIn(getManagementServerId()); + startWebSocketServer(); + long staleLogsWebSessionCleanupInterval = LogsWebServerSessionStaleCleanupInterval.value(); + staleLogsWebSessionCleanupExecutor.scheduleWithFixedDelay(new StaleLogsWebSessionCleanupWorker(), + staleLogsWebSessionCleanupInterval, staleLogsWebSessionCleanupInterval, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + stopWebSocketServer(1); + logsWebSessionDao.markAllActiveAsDisconnected(); + return true; + } + + @Override + public void startWebSocketServer() { + if (loggerWebSocketServer != null && loggerWebSocketServer.isRunning()) { + logger.info("Logger Web Socket Server is already running!"); + return; + } + loggerWebSocketServer = new LogsWebSocketServer(serverPort, serverPath, serverIdleTimeoutSeconds, + this); + try { + loggerWebSocketServer.start(); + } catch (InterruptedException e) { + logger.error("Failed to start Logger Web Socket Server", e); + } + } + + protected void stopWebSocketServer(Integer maxWaitSeconds) { + if (loggerWebSocketServer == null || !loggerWebSocketServer.isRunning()) { + logger.info("Logger Web Socket Server is already stopped!"); + return; + } + loggerWebSocketServer.stop(maxWaitSeconds == null ? 5 : maxWaitSeconds); + loggerWebSocketServer = null; + } + + @Override + public void stopWebSocketServer() { + stopWebSocketServer(null); + } + + private String getLogsWebSessionWebSocketPathUsingVO(long msId, LogsWebSession session) { + LogsWebSessionVO sessionVO = null; + if (session instanceof LogsWebSessionVO) { + sessionVO = (LogsWebSessionVO)session; + } else { + sessionVO = logsWebSessionDao.findById(session.getId()); + } + String path = serverPath; + if (!Objects.equals(msId, getManagementServerId())) { + serverPath = LogsWebServerPath.valueIn(msId); + } + return String.format("%s/%s", path, sessionVO.getUuid()); + } + + @Override + public List> getCommands() { + return List.of(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + LogsWebServerEnabled, + LogsWebServerPort, + LogsWebServerPath, + LogsWebServerConcurrentSessions, + LogsWebServerLogFile, + LogsWebServerSessionTailExistingLines, + LogsWebServerSessionIdleTimeout, + LogsWebServerSessionStaleCleanupInterval + }; + } + + @Override + public String getServerPath() { + return serverPath; + } + + @Override + public String getLogFile() { + return LogsWebServerLogFile.valueIn(getManagementServerId()); + } + + @Override + public int getMaxReadExistingLines() { + return LogsWebServerSessionTailExistingLines.valueIn(getManagementServerId()); + } + + @Override + public LogsWebSession getSession(String route) { + if (StringUtils.isBlank(route)) { + return null; + } + return logsWebSessionDao.findByUuid(route); + } + + @Override + public void updateSessionConnection(long sessionId, String clientAddress) { + LogsWebSessionVO logsWebSessionVO = logsWebSessionDao.findById(sessionId); + if (logsWebSessionVO == null) { + return; + } + if (StringUtils.isNotBlank(clientAddress)) { + logsWebSessionVO.setConnections(logsWebSessionVO.getConnections() + 1); + logsWebSessionVO.setConnectedTime(new Date()); + logsWebSessionVO.setClientAddress(clientAddress); + } else { + if (logsWebSessionVO.getConnections() == 0) { + return; + } + logsWebSessionVO.setConnections(Math.max(0, logsWebSessionVO.getConnections() - 1)); + } + logger.trace("Updating session: {}, is connected: {}, connections: {}", + logsWebSessionVO.getUuid(), + StringUtils.isBlank(clientAddress), + logsWebSessionVO.getConnections()); + logsWebSessionDao.update(sessionId, logsWebSessionVO); + } + + @Override + public List getLogsWebSessionWebSockets(final LogsWebSession logsWebSession) { + List webSockets = new ArrayList<>(); + final List activeMsList = + managementServerHostDao.listBy(ManagementServerHost.State.Up); + for (ManagementServerHostVO managementServerHostVO : activeMsList) { + LogsWebSessionWebSocket logsWebSessionWebSocket = new LogsWebSessionWebSocket(managementServerHostVO, + LogsWebServerPort.valueIn(managementServerHostVO.getId()), + getLogsWebSessionWebSocketPathUsingVO(managementServerHostVO.getId(), logsWebSession)); + webSockets.add(logsWebSessionWebSocket); + } + return webSockets; + } + + @Override + public boolean canCreateNewLogsWebSession() { + int maxSessions = LogsWebServerConcurrentSessions.valueIn(getManagementServerId()); + if (maxSessions <= 0) { + return true; + } + return maxSessions > logsWebSessionDao.countConnected(); + } + + public class StaleLogsWebSessionCleanupWorker extends ManagedContextRunnable { + + protected void runCleanupForStaleLogsWebSessions() { + try { + ManagementServerHostVO msHost = managementServerHostDao.findOneByLongestRuntime(); + if (msHost == null || (msHost.getMsid() != ManagementServerNode.getManagementServerId())) { + logger.debug("Skipping the stale logs web sessions cleanup task on this management server"); + return; + } + long cutOffSeconds = LogsWebServerSessionStaleCleanupInterval.value(); + Date cutOffDate = new Date(System.currentTimeMillis() - (cutOffSeconds * 1000)); + String cutOffDateString = DateUtil.getOutputString(cutOffDate); + logger.debug("Clearing stale stale logs web sessions older than {} using management server {}", + cutOffDateString, msHost); + long processed = logsWebSessionDao.removeStaleForCutOff(cutOffDate); + logger.debug("Cleared {} stale stale logs web sessions older than {}", processed, + cutOffDateString); + } catch (Exception e) { + logger.warn("Cleanup task failed to stale logs web sessions", e); + } + } + + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("LogsWebSessionsCleanup"); + try { + if (gcLock.lock(3)) { + try { + runCleanupForStaleLogsWebSessions(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionWebSocket.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionWebSocket.java new file mode 100644 index 000000000000..f897e8bb8499 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/LogsWebSessionWebSocket.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws; + +import com.cloud.cluster.ManagementServerHostVO; + +public class LogsWebSessionWebSocket { + + private ManagementServerHostVO managementServerHost; + private int port; + private String path; + + public LogsWebSessionWebSocket(final ManagementServerHostVO managementServerHost, final int port, + final String path) { + this.managementServerHost = managementServerHost; + this.port = port; + this.path = path; + } + + public ManagementServerHostVO getManagementServerHost() { + return managementServerHost; + } + + public int getPort() { + return port; + } + + public String getPath() { + return path; + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/CreateLogsWebSessionCmd.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/CreateLogsWebSessionCmd.java new file mode 100644 index 000000000000..2f7e8a7a607a --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/CreateLogsWebSessionCmd.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.api.command.admin; + + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.logsws.LogsWebSessionApiService; +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "createLogsWebSession", + description = "Creates a session to connect to logs web socket server", + responseObject = LogsWebSessionResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {LogsWebSession.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.21.0") +public class CreateLogsWebSessionCmd extends BaseCmd { + + @Inject + LogsWebSessionApiService logsWebSessionApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.FILTERS, type = CommandType.LIST, collectionType = CommandType.STRING, + description = "List of filter keywords") + private List filters; + + @Parameter(name = ApiConstants.TOKEN, type = CommandType.STRING, + description = "(Optional) extra security token, valid when the extra validation is enabled") + private String extraSecurityToken; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public List getFilters() { + return filters; + } + + public String getExtraSecurityToken() { + return extraSecurityToken; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + LogsWebSessionResponse response = logsWebSessionApiService.createLogsWebSession(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create logs web session"); + } + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } + +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/DeleteLogsWebSession.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/DeleteLogsWebSession.java new file mode 100644 index 000000000000..01ec709d9eba --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/DeleteLogsWebSession.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.api.command.admin; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.logsws.LogsWebSessionApiService; +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "deleteLogsWebSession", + description = "Deletes a logs web session", + responseObject = SuccessResponse.class, + entityType = {LogsWebSession.class}, + authorized = {RoleType.Admin}, + since = "4.21.0") +public class DeleteLogsWebSession extends BaseCmd { + + @Inject + LogsWebSessionApiService logsWebSessionApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, + entityType = LogsWebSessionResponse.class, + required = true, + description = "The ID of the logs web session") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + try { + if (!logsWebSessionApiService.deleteLogsWebSession(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + String.format("Failed to delete log web session ID: %d", getId())); + } + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/ListLogsWebSessionsCmd.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/ListLogsWebSessionsCmd.java new file mode 100644 index 000000000000..499f97cad26c --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/command/admin/ListLogsWebSessionsCmd.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.api.command.admin; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListAccountResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.logsws.LogsWebSessionApiService; +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.cloudstack.logsws.api.response.LogsWebSessionResponse; + +@APICommand(name = "listLogsWebSessions", + description = "Lists logs web sessions", + responseObject = LogsWebSessionResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {LogsWebSession.class}, + authorized = {RoleType.Admin}, + since = "4.21.0") +public class ListLogsWebSessionsCmd extends BaseListAccountResourcesCmd { + + @Inject + LogsWebSessionApiService logsWebSessionApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = LogsWebSessionResponse.class, + description = "The ID of the logs web session") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + ListResponse response = logsWebSessionApiService.listLogsWebSessions(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionResponse.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionResponse.java new file mode 100644 index 000000000000..3fcba9271ac9 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionResponse.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.api.response; + + +import java.util.Date; +import java.util.List; +import java.util.Set; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.api.response.ControlledViewEntityResponse; +import org.apache.cloudstack.logsws.LogsWebSession; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = {LogsWebSession.class}) +public class LogsWebSessionResponse extends BaseResponse implements ControlledViewEntityResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the logs web session") + private String id; + + @SerializedName(ApiConstants.FILTERS) + @Param(description = "The filters for the logs web session") + private List filters; + + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "The ID of the domain of the logs web session creator") + private String domainId; + + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "The name of the domain of the logs web session creator") + private String domainName; + + @SerializedName(ApiConstants.DOMAIN_PATH) + @Param(description = "The path of the domain of the logs web session creator") + private String domainPath; + + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "The account which created the logs web session") + private String accountName; + + @SerializedName(ApiConstants.CREATOR_ADDRESS) + @Param(description = "The address of creator for this logs web session") + private String creatorAddress; + + @SerializedName(ApiConstants.CONNECTED) + @Param(description = "The number of clients connected for this logs web session") + private Integer connected; + + @SerializedName(ApiConstants.CLIENT_ADDRESS) + @Param(description = "The address of the last connected client for this logs web session") + private String clientAddress; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "The date when this logs web session was created") + private Date created; + + @SerializedName(ApiConstants.WEBSOCKET) + @Param(description = "The logs web session websocket options") + private Set websocketResponses; + + public void setId(String id) { + this.id = id; + } + + public void setFilters(List filters) { + this.filters = filters; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + @Override + public void setDomainPath(String domainPath) { + this.domainPath = domainPath; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setProjectId(String projectId) { + } + + @Override + public void setProjectName(String projectName) { + } + + public void setCreatorAddress(String creatorAddress) { + this.creatorAddress = creatorAddress; + } + + public void setConnected(Integer connected) { + this.connected = connected; + } + + public void setClientAddress(String clientAddress) { + this.clientAddress = clientAddress; + } + + public void setCreated(Date created) { + this.created = created; + } + + public void setWebsocketResponse(Set websocketResponse) { + this.websocketResponses = websocketResponse; + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionWebSocketResponse.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionWebSocketResponse.java new file mode 100644 index 000000000000..60c2d0f9fcf8 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/api/response/LogsWebSessionWebSocketResponse.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class LogsWebSessionWebSocketResponse extends BaseResponse { + @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID) + @Param(description = "The ID of the management for this websocket") + private String managementServerId; + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME) + @Param(description = "The name of the management for this websocket") + private String managementServerName; + + @SerializedName("host") + @Param(description = "the websocket host") + private String host; + + @SerializedName(ApiConstants.PORT) + @Param(description = "the websocket port") + private Integer port; + + @SerializedName(ApiConstants.PATH) + @Param(description = "the websocket path") + private String path; + + public String getManagementServerId() { + return managementServerId; + } + + public void setManagementServerId(String managementServerId) { + this.managementServerId = managementServerId; + } + + public String getManagementServerName() { + return managementServerName; + } + + public void setManagementServerName(String managementServerName) { + this.managementServerName = managementServerName; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDao.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDao.java new file mode 100644 index 000000000000..2d9784c847ec --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDao.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.logsws.vo.LogsWebSessionVO; + +import com.cloud.utils.db.GenericDao; + +public interface LogsWebSessionDao extends GenericDao { + List listByAccount(long accountId); + void deleteByAccount(long accountId); + void markAllActiveAsDisconnected(); + int removeStaleForCutOff(Date cutOff); + int countConnected(); +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDaoImpl.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDaoImpl.java new file mode 100644 index 000000000000..95415f27a78d --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/dao/LogsWebSessionDaoImpl.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.dao; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.logsws.vo.LogsWebSessionVO; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.UpdateBuilder; + +public class LogsWebSessionDaoImpl extends GenericDaoBase implements LogsWebSessionDao { + SearchBuilder accountIdSearch; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + accountIdSearch = createSearchBuilder(); + accountIdSearch.and("accountId", accountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + + return true; + } + + @Override + public void deleteByAccount(long accountId) { + SearchCriteria sc = accountIdSearch.create(); + sc.setParameters("accountId", accountId); + remove(sc); + } + + @Override + public List listByAccount(long accountId) { + SearchCriteria sc = accountIdSearch.create(); + sc.setParameters("accountId", accountId); + return listBy(sc); + } + + @Override + public void markAllActiveAsDisconnected() { + SearchBuilder sb = createSearchBuilder(); + sb.and("connections", sb.entity().getConnections(), SearchCriteria.Op.GT); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("connections", 0); + LogsWebSessionVO logsWebSessionVO = createForUpdate(); + logsWebSessionVO.setConnections(0); + UpdateBuilder updateBuilder = getUpdateBuilder(logsWebSessionVO); + update(updateBuilder, sc, null); + } + + @Override + public int removeStaleForCutOff(Date cutOff) { + SearchBuilder sb = createSearchBuilder(); + sb.and("connections", sb.entity().getConnections(), SearchCriteria.Op.EQ); + sb.and().op("connected_time", sb.entity().getConnectedTime(), SearchCriteria.Op.LT); + sb.or().op("null_connected_time", sb.entity().getConnectedTime(), SearchCriteria.Op.NULL); + sb.and("created", sb.entity().getCreated(), SearchCriteria.Op.LT); + sb.cp(); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("connections", 0); + sc.setParameters("connected_time", cutOff); + sc.setParameters("created", cutOff); + return remove(sc); + } + + @Override + public int countConnected() { + GenericSearchBuilder sb = createSearchBuilder(Integer.class); + sb.and("connections", sb.entity().getConnections(), SearchCriteria.Op.GT); + sb.select(null, SearchCriteria.Func.COUNT, sb.entity().getId()); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("connections", 0); + return customSearch(sc, null).get(0); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/logreader/FilteredLogTailerListener.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/logreader/FilteredLogTailerListener.java new file mode 100644 index 000000000000..33c88815ef7b --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/logreader/FilteredLogTailerListener.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.logreader; + +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.input.TailerListenerAdapter; +import org.apache.commons.lang3.StringUtils; + +import io.netty.channel.Channel; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; + +public class FilteredLogTailerListener extends TailerListenerAdapter { + private final List filters; + private final Channel channel; + private final boolean isFilterEmpty; + private boolean isLastLineValid; + + public static boolean isValidLine(String line, boolean isFilterEmpty, + boolean isLastLineValid, List filters) { + if (StringUtils.isBlank(line)) { + return false; + } + if (isFilterEmpty) { + return true; + } + if (isLastLineValid && !line.startsWith("2025")) { + return true; + } + for (String filter : filters) { + if (line.contains(filter)) { + return true; + } + } + return false; + } + + public FilteredLogTailerListener(List filters, Channel channel) { + this.filters = filters; + this.channel = channel; + isFilterEmpty = CollectionUtils.isEmpty(filters); + isLastLineValid = false; + } + + @Override + public void handle(String line) { + // Check if the line contains the filter string + if (isValidLine(line, isFilterEmpty, isLastLineValid, filters)) { + channel.writeAndFlush(new TextWebSocketFrame(line)); + isLastLineValid = true; + } else { + isLastLineValid = false; + } + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketBroadcastHandler.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketBroadcastHandler.java new file mode 100644 index 000000000000..f9640715f4d1 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketBroadcastHandler.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.server; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.cloudstack.logsws.logreader.FilteredLogTailerListener; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.commons.io.input.Tailer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.util.ReferenceCountUtil; + +public class LogsWebSocketBroadcastHandler extends ChannelInboundHandlerAdapter { + protected static Logger LOGGER = LogManager.getLogger(LogsWebSocketBroadcastHandler.class); + private String route; + private Tailer tailer; + private ExecutorService tailerExecutor; + private final LogsWebSocketServerHelper serverHelper; + private LogsWebSession logsWebSession; + + public LogsWebSocketBroadcastHandler(final LogsWebSocketServerHelper serverHelper) { + this.serverHelper = serverHelper; + } + + private void startTestBroadcasting(final ChannelHandlerContext ctx) { + String route = ctx.channel().attr(LogsWebSocketRoutingHandler.LOGGER_ROUTE_ATTR).get(); + // Schedule a periodic task to send messages every 5 seconds. + ctx.executor().scheduleAtFixedRate(() -> { + if (ctx.channel().isActive()) { + String msg = String.format("Hello from Logger broadcaster! Route: %s", route); + ctx.writeAndFlush(new TextWebSocketFrame(msg)); + LOGGER.debug("Broadcasting message: '{}' for context: {}", msg, ctx.hashCode()); + } + }, 0, 5, TimeUnit.SECONDS); + } + + private void processExistingLines(final ChannelHandlerContext ctx, File logFile, final List filters) { + try (ReversedLinesFileReader reader = new ReversedLinesFileReader(logFile, StandardCharsets.UTF_8)) { + List lastLines = new ArrayList<>(); + String line; + int count = 0; + // Read lines in reverse order up to 200 lines + while ((line = reader.readLine()) != null && count < serverHelper.getMaxReadExistingLines()) { + lastLines.add(line); + count++; + } + // Reverse to restore original order + Collections.reverse(lastLines); + // Process each line that matches the filter + boolean isFilterEmpty = CollectionUtils.isEmpty(filters); + boolean isLastLineValid = false; + for (String l : lastLines) { + if (FilteredLogTailerListener.isValidLine(l, isFilterEmpty, isLastLineValid, filters)) { + ctx.writeAndFlush(new TextWebSocketFrame(l)); + isLastLineValid = true; + } else { + isLastLineValid = false; + } + } + } catch (IOException e) { + ctx.writeAndFlush(new TextWebSocketFrame("Error reading existing log lines: " + e.getMessage())); + } + } + + private void startLogTailing(ChannelHandlerContext ctx, List filters, File logFile) { + // Create the listener to filter new log lines + FilteredLogTailerListener listener = new FilteredLogTailerListener(filters, ctx.channel()); + // Use 'true' to start tailing from the end of the file (since we've already processed existing lines) + tailer = new Tailer(logFile, listener, 100, true); + + // Use an executor service for managing the tailer thread + tailerExecutor = Executors.newSingleThreadExecutor(); + tailerExecutor.submit(tailer); + } + + private void startLogsBroadcasting(final ChannelHandlerContext ctx) { + route = ctx.channel().attr(LogsWebSocketRoutingHandler.LOGGER_ROUTE_ATTR).get(); + logsWebSession = serverHelper.getSession(route); + if (logsWebSession == null) { + LOGGER.warn("Unauthorized session for route: {}", route); + ctx.close(); + return; + } + File logFile = new File(serverHelper.getLogFile()); + if (!logFile.exists() || !logFile.canRead()) { + ctx.channel().writeAndFlush(new TextWebSocketFrame("Log file not available or cannot be read.")); + return; + } + InetSocketAddress clientAddress = (InetSocketAddress) ctx.channel().remoteAddress(); + serverHelper.updateSessionConnection(logsWebSession.getId(), clientAddress.getAddress().getHostAddress()); + processExistingLines(ctx, logFile, logsWebSession.getFilters()); + startLogTailing(ctx, logsWebSession.getFilters(), logFile); + } + + private void stopLogsBroadcasting() { + if (tailer != null) { + tailer.stop(); + } + if (tailerExecutor != null && !tailerExecutor.isShutdown()) { + tailerExecutor.shutdownNow(); + } + if (logsWebSession != null) { + serverHelper.updateSessionConnection(logsWebSession.getId(), null); + } + } + + @Override + public void channelActive(final ChannelHandlerContext ctx) throws Exception { + LOGGER.debug("Channel is active, context: {}", ctx.hashCode()); + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + LOGGER.debug("Channel is being closed for route: {}, context: {}", route, ctx.hashCode()); + stopLogsBroadcasting(); + super.channelInactive(ctx); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + LOGGER.debug("User event triggered: {}, context: {}", evt, ctx.hashCode()); + if (evt instanceof WebSocketServerProtocolHandler.HandshakeComplete) { + startLogsBroadcasting(ctx); + } else if (evt instanceof IdleStateEvent) { + IdleStateEvent event = (IdleStateEvent) evt; + if (IdleState.WRITER_IDLE.equals(event.state())) { + ctx.channel().writeAndFlush(new TextWebSocketFrame("Connection idle for 1 minute, closing connection.")); + ctx.close(); + return; + } + } + super.userEventTriggered(ctx, evt); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + // Discard any messages received from the client. + ReferenceCountUtil.release(msg); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRouteManager.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRouteManager.java new file mode 100644 index 000000000000..be47b9457044 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRouteManager.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.server; + +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import io.netty.channel.group.ChannelGroup; +import io.netty.channel.group.DefaultChannelGroup; +import io.netty.util.concurrent.GlobalEventExecutor; + +public class LogsWebSocketRouteManager { + protected static Logger LOGGER = LogManager.getLogger(LogsWebSocketRouteManager.class); + private final ConcurrentHashMap routeMap = new ConcurrentHashMap<>(); + + public void addRoute(String route) { + routeMap.putIfAbsent(route, new DefaultChannelGroup(GlobalEventExecutor.INSTANCE)); + LOGGER.debug("Added route: {}", route); + } + + public void removeRoute(String route) { + ChannelGroup group = routeMap.remove(route); + if (group == null) { + LOGGER.debug("Route: {} doesn't exist", route); + return; + } + group.close(); + LOGGER.debug("Removed route: {}", route); + } + + public ChannelGroup getRouteGroup(String route) { + return routeMap.get(route); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRoutingHandler.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRoutingHandler.java new file mode 100644 index 000000000000..b2c69fb08dfb --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketRoutingHandler.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.server; + +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.group.ChannelGroup; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.util.AttributeKey; + +public class LogsWebSocketRoutingHandler extends ChannelInboundHandlerAdapter { + protected static Logger LOGGER = LogManager.getLogger(LogsWebSocketRoutingHandler.class); + public static final AttributeKey LOGGER_ROUTE_ATTR = AttributeKey.valueOf("loggerRoute"); + private final LogsWebSocketRouteManager routeManager; + private final LogsWebSocketServerHelper serverHelper; + + public LogsWebSocketRoutingHandler(LogsWebSocketRouteManager routeManager, + LogsWebSocketServerHelper serverHelper) { + this.routeManager = routeManager; + this.serverHelper = serverHelper; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (!(msg instanceof FullHttpRequest)) { + ctx.fireChannelRead(msg); + return; + } + FullHttpRequest req = (FullHttpRequest) msg; + String uri = req.uri(); + LOGGER.debug("Original URI: {}", uri); + final String serverPath = serverHelper.getServerPath(); + final String expectedPathPrefix = serverPath + "/"; + if (!uri.startsWith(expectedPathPrefix)) { + ctx.close(); + return; + } + // Extract the route portion. + String route = uri.substring(expectedPathPrefix.length()); + if (route.isEmpty()) { + ctx.close(); + return; + } + + LogsWebSession session = serverHelper.getSession(route); + if (session == null) { + LOGGER.warn("Unauthorized connection attempt for route: {}", route); + ctx.close(); + return; + } + // Retrieve or add the route. + ChannelGroup group = routeManager.getRouteGroup(route); + if (group == null) { + routeManager.addRoute(route); + group = routeManager.getRouteGroup(route); + } else { + // If there's already a connection, close it to allow only one connection per route. + if (!group.isEmpty()) { + LOGGER.debug("Closing existing connection(s) for route: {}", route); + group.close(); // This will close all existing channels in the group. + } + } + + LOGGER.debug("Connecting to route: {} for context: {}", route, ctx.hashCode()); + ctx.channel().attr(LOGGER_ROUTE_ATTR).set(route); + group.add(ctx.channel()); + + // Rewrite the URI so that the handshake matches the expected sever path + if (req instanceof DefaultFullHttpRequest) { + ((DefaultFullHttpRequest) req).setUri(serverPath); + } else { + DefaultFullHttpRequest newReq = new DefaultFullHttpRequest( + req.protocolVersion(), req.method(), serverPath, req.content().retain()); + newReq.headers().setAll(req.headers()); + req.release(); + req = newReq; + } + LOGGER.debug("Rewritten URI: {}", req.uri()); + ctx.fireChannelRead(req); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + LOGGER.error("Exception in LoggerWebSocketRoutingHandler", cause); + ctx.close(); + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServer.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServer.java new file mode 100644 index 000000000000..fb2965ab1960 --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServer.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.server; + +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler; +import io.netty.handler.timeout.IdleStateHandler; + +public class LogsWebSocketServer { + + protected static Logger LOGGER = LogManager.getLogger(LogsWebSocketServer.class); + + private final int port; + private final String path; + private final int idleTimeoutSeconds; + private EventLoopGroup bossGroup; + private EventLoopGroup workerGroup; + private Channel serverChannel; + private boolean running; + private final LogsWebSocketRouteManager routeManager; + private final LogsWebSocketServerHelper serverHelper; + + public LogsWebSocketServer(final int port, final String path, final int idleTimeoutSeconds, + final LogsWebSocketServerHelper serverHelper) { + this.port = port; + this.path = path; + this.idleTimeoutSeconds = idleTimeoutSeconds; + this.serverHelper = serverHelper; + this.routeManager = new LogsWebSocketRouteManager(); + } + + public void start() throws InterruptedException { + bossGroup = new NioEventLoopGroup(1); + workerGroup = new NioEventLoopGroup(); + ServerBootstrap b = new ServerBootstrap(); + b.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ChannelPipeline pipeline = ch.pipeline(); + pipeline.addLast(new HttpServerCodec()); + pipeline.addLast(new HttpObjectAggregator(65536)); + pipeline.addLast(new LogsWebSocketRoutingHandler(routeManager, serverHelper)); + pipeline.addLast(new WebSocketServerProtocolHandler(path, null, true)); + pipeline.addLast("idleStateHandler", new IdleStateHandler(0, idleTimeoutSeconds, 0, TimeUnit.SECONDS)); + pipeline.addLast(new LogsWebSocketBroadcastHandler(serverHelper)); + } + }); + + // Bind and store the server channel. + serverChannel = b.bind(port).sync().channel(); + LOGGER.debug("Logger WebSocket server started on port {}", port); + // Note: We do not block here with serverChannel.closeFuture().sync() + running = true; + } + + // Stop the server gracefully. + public void stop() { + stop(5); + } + + public void stop(long maxWaitSeconds) { + try { + if (serverChannel != null) { + serverChannel.close().sync(); + } + if (bossGroup != null) { + bossGroup.shutdownGracefully(0, maxWaitSeconds, TimeUnit.SECONDS).sync(); + } + if (workerGroup != null) { + workerGroup.shutdownGracefully(0, maxWaitSeconds, TimeUnit.SECONDS).sync(); + } + } catch (InterruptedException e) { + LOGGER.error("Failed to stop WebSocket server properly with timeout {}s, forcefully stopping", + maxWaitSeconds, e); + if (serverChannel != null && serverChannel.isOpen()) { + serverChannel.close(); + } + if (bossGroup != null && !bossGroup.isTerminated()) { + bossGroup.shutdownGracefully(0, 0, TimeUnit.SECONDS); + } + if (workerGroup != null && !workerGroup.isTerminated()) { + workerGroup.shutdownGracefully(0, 0, TimeUnit.SECONDS); + } + } + LOGGER.debug("Logger WebSocket server stopped"); + running = false; + } + + public boolean isRunning() { + return running; + } +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServerHelper.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServerHelper.java new file mode 100644 index 000000000000..c630f3a4d26a --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/server/LogsWebSocketServerHelper.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.server; + +import org.apache.cloudstack.logsws.LogsWebSession; + +public interface LogsWebSocketServerHelper { + String getServerPath(); + String getLogFile(); + int getMaxReadExistingLines(); + LogsWebSession getSession(String route); + void updateSessionConnection(long sessionId, String clientAddress); +} diff --git a/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/vo/LogsWebSessionVO.java b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/vo/LogsWebSessionVO.java new file mode 100644 index 000000000000..e1cef92b206a --- /dev/null +++ b/plugins/logs-web-server/src/main/java/org/apache/cloudstack/logsws/vo/LogsWebSessionVO.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.logsws.vo; + + +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.logsws.LogsWebSession; +import org.apache.cloudstack.util.StringListJsonConverter; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "logs_web_session") +public class LogsWebSessionVO implements LogsWebSession { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "filter", columnDefinition = "json") + @Convert(converter = StringListJsonConverter.class) + private List filters; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "creator_address") + private String creatorAddress; + + @Column(name = "connections") + private int connections; + + @Column(name = "connected_time") + @Temporal(value = TemporalType.TIMESTAMP) + private Date connectedTime; + + @Column(name = "client_address") + private String clientAddress; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public List getFilters() { + return filters; + } + + @Override + public long getDomainId() { + return domainId; + } + + public void setDomainId(long domainId) { + this.domainId = domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + @Override + public int getConnections() { + return connections; + } + + public void setConnections(int connections) { + this.connections = connections; + } + + @Override + public Date getConnectedTime() { + return connectedTime; + } + + public void setConnectedTime(Date connectedTime) { + this.connectedTime = connectedTime; + } + + @Override + public String getCreatorAddress() { + return creatorAddress; + } + + public void setCreatorAddress(String creatorAddress) { + this.creatorAddress = creatorAddress; + } + + @Override + public String getClientAddress() { + return clientAddress; + } + + public void setClientAddress(String clientAddress) { + this.clientAddress = clientAddress; + } + + @Override + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public LogsWebSessionVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public LogsWebSessionVO(List filters, long domainId, long accountId, String creatorAddress) { + this.filters = filters; + this.uuid = UUID.randomUUID().toString(); + this.domainId = domainId; + this.accountId = accountId; + this.creatorAddress = creatorAddress; + } + + @Override + public Class getEntityType() { + return LogsWebSession.class; + } + + @Override + public String getName() { + return uuid; + } +} diff --git a/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/module.properties b/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/module.properties new file mode 100644 index 000000000000..1839e8819b2f --- /dev/null +++ b/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=logs-web-server +parent=backend diff --git a/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/spring-logs-web-server-context.xml b/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/spring-logs-web-server-context.xml new file mode 100644 index 000000000000..dc42491d9570 --- /dev/null +++ b/plugins/logs-web-server/src/main/resources/META-INF/cloudstack/logs-web-server/spring-logs-web-server-context.xml @@ -0,0 +1,33 @@ + + + + + + + diff --git a/plugins/logs-web-server/src/test/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImplTest.java b/plugins/logs-web-server/src/test/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImplTest.java new file mode 100644 index 000000000000..c8a2040346d1 --- /dev/null +++ b/plugins/logs-web-server/src/test/java/org/apache/cloudstack/logsws/LogsWebSessionManagerImplTest.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.logsws; + +import java.util.UUID; + +import org.apache.cloudstack.logsws.dao.LogsWebSessionDao; +import org.apache.cloudstack.logsws.vo.LogsWebSessionVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class LogsWebSessionManagerImplTest { + + @Mock + LogsWebSessionDao logsWebSessionDao; + + @InjectMocks + LogsWebSessionManagerImpl logsWSManager = new LogsWebSessionManagerImpl(); + + @Test + public void test_getSession_nullRoute() { + Assert.assertNull(logsWSManager.getSession(null)); + Assert.assertNull(logsWSManager.getSession("abc")); + } + + @Test + public void test_getSession_validRoute() { + String uuid = UUID.randomUUID().toString(); + Mockito.when(logsWebSessionDao.findByUuid(uuid)).thenReturn(Mockito.mock(LogsWebSessionVO.class)); + Assert.assertNotNull(logsWSManager.getSession(uuid)); + } +} \ No newline at end of file diff --git a/plugins/pom.xml b/plugins/pom.xml index e7d13871285e..48eddeaae438 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -97,6 +97,8 @@ integrations/prometheus integrations/kubernetes-service + logs-web-server + metrics network-elements/bigswitch diff --git a/pom.xml b/pom.xml index 2a36c1cc4efa..0fd2638932f3 100644 --- a/pom.xml +++ b/pom.xml @@ -188,6 +188,7 @@ 5.3.26 0.5.4 3.1.7 + 4.1.95.Final @@ -735,6 +736,11 @@ java-linstor ${cs.java-linstor.version} + + io.netty + netty-all + ${cs.netty.all.version} + diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index c78ac05102f8..98fd308cdf71 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.api; +import static com.cloud.user.AccountManagerImpl.apiKeyAccess; +import static org.apache.cloudstack.user.UserPasswordResetManager.UserPasswordResetEnabled; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InterruptedIOException; @@ -57,15 +60,6 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.dao.ManagementServerHostDao; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountManagerImpl; -import com.cloud.user.DomainManager; -import com.cloud.user.User; -import com.cloud.user.UserAccount; -import com.cloud.user.UserVO; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -105,6 +99,7 @@ import org.apache.cloudstack.api.response.LoginCmdResponse; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.context.LogContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.events.EventDistributor; @@ -154,6 +149,8 @@ import com.cloud.api.dispatch.DispatchChainFactory; import com.cloud.api.dispatch.DispatchTask; import com.cloud.api.response.ApiResponseSerializer; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; @@ -172,14 +169,22 @@ import com.cloud.exception.UnavailableCommandException; import com.cloud.projects.dao.ProjectDao; import com.cloud.storage.VolumeApiService; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountManagerImpl; +import com.cloud.user.DomainManager; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.UserVO; import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.DateUtil; import com.cloud.utils.HttpUtils; -import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.Pair; import com.cloud.utils.ReflectUtil; import com.cloud.utils.StringUtils; +import com.cloud.utils.UuidUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -192,9 +197,6 @@ import com.cloud.utils.net.NetUtils; import com.google.gson.reflect.TypeToken; -import static com.cloud.user.AccountManagerImpl.apiKeyAccess; -import static org.apache.cloudstack.user.UserPasswordResetManager.UserPasswordResetEnabled; - @Component public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiServerService, Configurable { private static final Logger ACCESSLOGGER = LogManager.getLogger("apiserver." + ApiServer.class.getName()); @@ -735,12 +737,17 @@ public boolean isPostRequestsAndTimestampsEnforced() { return isPostRequestsAndTimestampsEnforced; } + private String getCurrentContextId() { + return UuidUtils.first(LogContext.current().getLogContextId()); + } + private String getBaseAsyncResponse(final long jobId, final BaseAsyncCmd cmd) { final AsyncJobResponse response = new AsyncJobResponse(); final AsyncJob job = entityMgr.findByIdIncludingRemoved(AsyncJob.class, jobId); response.setJobId(job.getUuid()); response.setResponseName(cmd.getCommandName()); + response.setContextId(getCurrentContextId()); return ApiResponseSerializer.toSerializedString(response, cmd.getResponseType()); } @@ -750,6 +757,7 @@ private String getBaseAsyncCreateResponse(final long jobId, final BaseAsyncCreat response.setJobId(job.getUuid()); response.setId(objectUuid); response.setResponseName(cmd.getCommandName()); + response.setContextId(getCurrentContextId()); return ApiResponseSerializer.toSerializedString(response, cmd.getResponseType()); } @@ -867,7 +875,9 @@ private String queueCommand(final BaseCmd cmdObj, final Map para } SerializationContext.current().setUuidTranslation(true); - return ApiResponseSerializer.toSerializedStringWithSecureLogs((ResponseObject)cmdObj.getResponseObject(), cmdObj.getResponseType(), log); + ResponseObject responseObject = (ResponseObject)cmdObj.getResponseObject(); + responseObject.setContextId(getCurrentContextId()); + return ApiResponseSerializer.toSerializedStringWithSecureLogs(responseObject, cmdObj.getResponseType(), log); } } @@ -902,6 +912,7 @@ private void buildAsyncListResponse(final BaseListCmd command, final Account acc final AsyncJob job = objectJobMap.get(response.getObjectId()); response.setJobId(job.getUuid()); response.setJobStatus(job.getStatus().ordinal()); + response.setContextId(getCurrentContextId()); } } } diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index 21d093758127..7a37b8f86f53 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -25,8 +25,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import java.util.Set; +import java.util.regex.Pattern; import javax.inject.Inject; import javax.servlet.ServletConfig; @@ -52,10 +52,9 @@ import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.utils.consoleproxy.ConsoleAccessUtils; import org.apache.commons.collections.MapUtils; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.EnumUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; @@ -70,10 +69,9 @@ import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.user.UserAccount; - import com.cloud.utils.HttpUtils; -import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.StringUtils; import com.cloud.utils.db.EntityManager; import com.cloud.utils.net.NetUtils; @@ -117,6 +115,7 @@ public class ApiServlet extends HttpServlet { "listandswitchsamlaccount", "uploadresourceicon" )); + public static final String CLIENT_INET_ADDRESS_KEY = "client-inet-address"; @Inject ApiServerService apiServer; diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 3f811c152f00..beafbea348e9 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -789,7 +789,6 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.GuestOsCategory; import com.cloud.storage.ScopeType; -import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -806,6 +805,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.secondary.SecondaryStorageVmManager; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateManager; @@ -4710,6 +4710,7 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { final boolean kubernetesServiceEnabled = Boolean.parseBoolean(_configDao.getValue("cloud.kubernetes.service.enabled")); final boolean kubernetesClusterExperimentalFeaturesEnabled = Boolean.parseBoolean(_configDao.getValue("cloud.kubernetes.cluster.experimental.features.enabled")); + final boolean logsWebServerEnabled = Boolean.parseBoolean(_configDao.getValue("logs.web.server.enabled")); // check if region-wide secondary storage is used boolean regionSecondaryEnabled = false; @@ -4758,6 +4759,7 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { capabilities.put(ApiConstants.EXTENSIONS_PATH, extensionsManager.getExtensionsPath()); } capabilities.put(ApiConstants.ADDITONAL_CONFIG_ENABLED, UserVmManager.EnableAdditionalVmConfig.valueIn(caller.getId())); + capabilities.put(ApiConstants.LOGS_WEB_SERVER_ENABLED, logsWebServerEnabled); return capabilities; } diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index e41a04ff2e1b..f5877a1dd0c9 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -273,7 +273,9 @@ 'Extensions' : 'Extension', 'CustomAction' : 'Extension', 'CustomActions' : 'Extension', - 'ImportVmTask': 'Import VM Task' + 'ImportVmTask': 'Import VM Task', + 'LogsWebSession': 'Logs Web Session', + 'LogsWebSessions': 'Logs Web Session' } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 805ea1adae94..21f9646fd5c3 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2693,6 +2693,7 @@ "label.view": "View", "label.view.all": "View all", "label.view.console": "View console", +"label.view.logs": "View logs", "label.viewing": "Viewing", "label.virtualmachine": "Instance", "label.virtualmachinecount": "Instances Count", @@ -3771,6 +3772,7 @@ "message.setup.physical.network.during.zone.creation.basic": "When adding a basic Zone, you can set up one physical Network, which corresponds to a NIC on the hypervisor. The Network carries several types of traffic.

You may also add other traffic types onto the physical Network.", "message.shared.network.offering.warning": "Domain admins and regular Users can only create shared Networks from Network offering with the setting specifyvlan=false. Please contact an administrator to create a Network offering if this list is empty.", "message.shared.network.unsupported.for.nsx": "Shared networks aren't supported for NSX enabled Zones", +"message.showing.logs": "Showing logs for '%x'", "message.shutdown.triggered": "A shutdown has been triggered. CloudStack will not accept new jobs", "message.snapshot.additional.zones": "Snapshots will always be created in its native Zone - %x, here you can select additional zone(s) where it will be copied to at creation time", "message.snapshot.desc": "Snapshot to create a ROOT disk from", diff --git a/ui/src/components/page/GlobalLayout.vue b/ui/src/components/page/GlobalLayout.vue index 5c37c6f83d66..e5adfbfbe60d 100644 --- a/ui/src/components/page/GlobalLayout.vue +++ b/ui/src/components/page/GlobalLayout.vue @@ -70,6 +70,11 @@ + + +