extensions: add sync functionality

This change introduces support for synchronizing extension files across management servers in a clustered environment.

- Adds new syncExtension API to trigger synchronization for a selected extension.
- Implements service to create .tgz archive of the extension directory or selected files.
- Generates signed share URL with HMAC signature and expiry.
- Sends DownloadAndSyncExtensionFilesCommand to peer management servers.
- Handles archive download, staging, and extraction on receiver side.
- Adds Sync Extension action in the UI Extensions view.
- Updates events, logging, and cleanup of temporary share files.

Refactors all filesystem related code to extensions framework layer.
Checksum for extensions is now calculated and compared for all files in
the extension directory.

Signed-off-by: Abhishek Kumar <[email protected]>

Author: shwstppr

api/src/main/java/com/cloud/event/EventTypes.java

@@ -844,6 +844,7 @@ public class EventTypes {
     public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE";
     public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE";
     public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
+    public static final String EVENT_EXTENSION_SYNC = "EXTENSION.SYNC";
     public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
     public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
     public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
@@ -1385,6 +1386,7 @@ public class EventTypes {
         entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_SYNC, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class);

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -547,6 +547,7 @@ public class ApiConstants {
 
     public static final String SOURCE_CIDR_LIST = "sourcecidrlist";
     public static final String SOURCE_ZONE_ID = "sourcezoneid";
+    public static final String SOURCE_MANAGEMENT_SERVER_ID = "sourcemanagementserverid";
     public static final String SSL_VERIFICATION = "sslverification";
     public static final String START_ASN = "startasn";
     public static final String START_DATE = "startdate";
@@ -567,6 +568,7 @@ public class ApiConstants {
     public static final String SWAP_OWNER = "swapowner";
     public static final String SYSTEM_VM_TYPE = "systemvmtype";
     public static final String TAGS = "tags";
+    public static final String TARGET_MANAGEMENT_SERVER_IDS = "targetmanagementserverids";
     public static final String STORAGE_TAGS = "storagetags";
     public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups";
     public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup";

client/conf/server.properties.in

@@ -62,3 +62,19 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@
 # Thread pool configuration
 #threads.min=10
 #threads.max=500
+
+# These properties configure the share endpoint, which enables controlled file sharing through the management server.
+# They allow administrators to enable or disable sharing, set the base directory for shared files, define cache
+# behavior, restrict access to specific directories, and secure access with a secret key. This ensures flexible and
+# secure file sharing for different modules such as extensions, etc.
+# Enable or disable file sharing feature (true/false). Default is true
+share.enabled=true
+# The base directory from which files can be shared. Default is <HOME_DIRECTORY_OF_CLOUD_USER>/share
+# share.base.dir=
+# The cache control header value to be used for shared files. Default is public,max-age=86400,immutable
+# share.cache.control=public,max-age=86400,immutable
+# Allow or disallow directory listing when accessing a directory. Default is false
+# share.dir.allowed=false
+# Secret key for securing links using HMAC signature. If not set then links will not be signed. Default is change-me
+# It is recommended to change this value to a strong secret key in production
+share.secret=change-me

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -24,15 +24,25 @@
 import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
 import java.util.Properties;
 
-import com.cloud.api.ApiServer;
+import javax.servlet.DispatcherType;
+
+import org.apache.cloudstack.utils.server.ServerPropertiesUtil;
 import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.eclipse.jetty.jmx.MBeanContainer;
 import org.eclipse.jetty.server.ForwardedRequestCustomizer;
+import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.RequestLog;
@@ -46,14 +56,18 @@
 import org.eclipse.jetty.server.handler.RequestLogHandler;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.ssl.KeyStoreScanner;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
 import org.eclipse.jetty.webapp.WebAppContext;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 
+import com.cloud.api.ApiServer;
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.server.ServerProperties;
@@ -111,6 +125,12 @@ public class ServerDaemon implements Daemon {
     private int minThreads;
     private int maxThreads;
 
+    private boolean shareEnabled = false;
+    private String shareBaseDir;
+    private String shareCacheCtl;
+    private boolean shareDirList = false;
+    private String shareSecret;
+
     //////////////////////////////////////////////////
     /////////////// Public methods ///////////////////
     //////////////////////////////////////////////////
@@ -121,6 +141,14 @@ public static void main(final String... anArgs) throws Exception {
         daemon.start();
     }
 
+    protected void initShareConfigFromProperties() {
+        setShareEnabled(ServerPropertiesUtil.getShareEnabled());
+        setShareBaseDir(ServerPropertiesUtil.getShareBaseDirectory());
+        setShareCacheCtl(ServerPropertiesUtil.getShareCacheControl());
+        setShareDirList(ServerPropertiesUtil.getShareDirAllowed());
+        setShareSecret(ServerPropertiesUtil.getShareSecret());
+    }
+
     @Override
     public void init(final DaemonContext context) {
         final File confFile = PropertiesUtil.findConfigFile("server.properties");
@@ -153,6 +181,7 @@ public void init(final DaemonContext context) {
             setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
             setMinThreads(Integer.valueOf(properties.getProperty(THREADS_MIN, "10")));
             setMaxThreads(Integer.valueOf(properties.getProperty(THREADS_MAX, "500")));
+            initShareConfigFromProperties();
         } catch (final IOException e) {
             logger.warn("Failed to read configuration from server.properties file", e);
         } finally {
@@ -288,6 +317,52 @@ private void createHttpsConnector(final HttpConfiguration httpConfig) {
         }
     }
 
+    /**
+     * Creates a Jetty context at /share to serve static files for modules (e.g. Extensions Framework).
+     * Controlled via server properties
+     *
+     * @return a configured Handler or null if disabled.
+     */
+    private Handler createShareContextHandler() throws IOException {
+        if (!shareEnabled) {
+            logger.info("/{} context not mounted", ServerPropertiesUtil.SHARE_DIR);
+            return null;
+        }
+
+        final Path base = Paths.get(shareBaseDir);
+        Files.createDirectories(base);
+
+        final ServletContextHandler shareCtx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+        shareCtx.setContextPath("/" + ServerPropertiesUtil.SHARE_DIR);
+        shareCtx.setBaseResource(Resource.newResource(base.toAbsolutePath().toUri()));
+
+        // Efficient static file serving
+        ServletHolder def = shareCtx.addServlet(DefaultServlet.class, "/*");
+        def.setInitParameter("dirAllowed", Boolean.toString(shareDirList));
+        def.setInitParameter("etags", "true");
+        def.setInitParameter("cacheControl", shareCacheCtl);
+        def.setInitParameter("useFileMappedBuffer", "true");
+        def.setInitParameter("acceptRanges", "true");
+
+        // Gzip using modern Jetty handler
+        org.eclipse.jetty.server.handler.gzip.GzipHandler gzipHandler =
+                new org.eclipse.jetty.server.handler.gzip.GzipHandler();
+        gzipHandler.setMinGzipSize(1024);
+        gzipHandler.setIncludedMimeTypes(
+                "text/html", "text/plain", "text/css", "text/javascript",
+                "application/javascript", "application/json", "application/xml");
+        gzipHandler.setHandler(shareCtx);
+
+        // Optional signed-URL guard (path + "|" + exp => HMAC-SHA256, base64url)
+        if (StringUtils.isNotBlank(shareSecret)) {
+            shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(true, shareSecret)),
+                    "/*", EnumSet.of(DispatcherType.REQUEST));
+        }
+
+        logger.info("Mounted /{} static context at baseDir={}", ServerPropertiesUtil.SHARE_DIR, base);
+        return shareCtx;
+    }
+
     private Pair<SessionHandler,HandlerCollection> createHandlers() {
         final WebAppContext webApp = new WebAppContext();
         webApp.setContextPath(contextPath);
@@ -318,8 +393,23 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
         rootRedirect.setNewContextURL(contextPath);
         rootRedirect.setPermanent(true);
 
+        // Optional /share handler (served by createShareContextHandler)
+        Handler shareHandler = null;
+        try {
+            shareHandler = createShareContextHandler();
+        } catch (IOException e) {
+            logger.error("Failed to initialize /share context", e);
+        }
+
+        List<Handler> handlers = new java.util.ArrayList<>();
+        handlers.add(log);
+        handlers.add(gzipHandler);
+        if (shareHandler != null) {
+            handlers.add(shareHandler);
+        }
         // Put rootRedirect at the end!
-        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect));
+        handlers.add(rootRedirect);
+        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(handlers.toArray(new Handler[0])));
     }
 
     private RequestLog createRequestLog() {
@@ -408,4 +498,24 @@ public void setMinThreads(int minThreads) {
     public void setMaxThreads(int maxThreads) {
         this.maxThreads = maxThreads;
     }
+
+    public void setShareEnabled(boolean shareEnabled) {
+        this.shareEnabled = shareEnabled;
+    }
+
+    public void setShareBaseDir(String shareBaseDir) {
+        this.shareBaseDir = shareBaseDir;
+    }
+
+    public void setShareCacheCtl(String shareCacheCtl) {
+        this.shareCacheCtl = shareCacheCtl;
+    }
+
+    public void setShareDirList(boolean shareDirList) {
+        this.shareDirList = shareDirList;
+    }
+
+    public void setShareSecret(String shareSecret) {
+        this.shareSecret = shareSecret;
+    }
 }

client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java

@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack;
+
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.cloudstack.utils.security.HMACSignUtil;
+import org.apache.commons.codec.DecoderException;
+
+/**
+ * Optional HMAC token check: /share/...?...&exp=1699999999&sig=BASE64URL(HMACSHA256(path|exp))
+ */
+public class ShareSignedUrlFilter implements Filter {
+    private final boolean requireToken;
+    private final String secret;
+
+    public ShareSignedUrlFilter(boolean requireToken, String secret) {
+        this.requireToken = requireToken;
+        this.secret = secret;
+    }
+
+    @Override public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
+            throws IOException, ServletException {
+        HttpServletRequest r = (HttpServletRequest) req;
+        HttpServletResponse w = (HttpServletResponse) res;
+
+        String expStr = r.getParameter("exp");
+        String sig = r.getParameter("sig");
+
+        if (!requireToken && (expStr == null || sig == null)) {
+            chain.doFilter(req, res);
+            return;
+        }
+        if (expStr == null || sig == null) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token");
+            return;
+        }
+        long exp;
+        try {
+            exp = Long.parseLong(expStr);
+        } catch (NumberFormatException e) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp");
+            return;
+        }
+        if (Instant.now().getEpochSecond() > exp) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired");
+            return;
+        }
+        String want = "";
+        try {
+            String data = r.getRequestURI() + "|" + expStr;
+            want = HMACSignUtil.generateSignature(data, secret);
+        } catch (InvalidKeyException | NoSuchAlgorithmException | DecoderException e) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Auth error");
+            return;
+        }
+        if (!want.equals(sig)) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature");
+            return;
+        }
+        chain.doFilter(req, res);
+    }
+}

engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java

@@ -35,18 +35,6 @@
 
 public interface ExternalProvisioner extends Manager {
 
-    String getExtensionsPath();
-
-    String getExtensionPath(String relativePath);
-
-    String getChecksumForExtensionPath(String extensionName, String relativePath);
-
-    void prepareExtensionPath(String extensionName, boolean userDefined, String extensionRelativePath);
-
-    void cleanupExtensionPath(String extensionName, String extensionRelativePath);
-
-    void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory);
-
     PrepareExternalProvisioningAnswer prepareExternalProvisioning(String hostGuid, String extensionName, String extensionRelativePath, PrepareExternalProvisioningCommand cmd);
 
     StartAnswer startInstance(String hostGuid, String extensionName, String extensionRelativePath, StartCommand cmd);

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -468,6 +468,8 @@ public String execute(final String strPeer, final long agentId, final String cmd
         return null;
     }
 
+
+
     @Override
     public ManagementServerHostVO getPeer(final String mgmtServerId) {
         return _mshostDao.findByMsid(Long.parseLong(mgmtServerId));

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java

@@ -61,4 +61,6 @@ public interface ManagementServerHostDao extends GenericDao<ManagementServerHost
     ManagementServerHostVO findOneInUpState(Filter filter);
 
     ManagementServerHostVO findOneByLongestRuntime();
+
+    List<ManagementServerHostVO> listUpByIds(List<Long> ids);
 }

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java

@@ -24,12 +24,11 @@
 import java.util.List;
 import java.util.TimeZone;
 
-
+import org.apache.cloudstack.management.ManagementServerHost;
+import org.apache.cloudstack.management.ManagementServerHost.State;
 import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.cluster.ClusterInvalidSessionException;
-import org.apache.cloudstack.management.ManagementServerHost;
-import org.apache.cloudstack.management.ManagementServerHost.State;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.DB;
@@ -318,4 +317,18 @@ public ManagementServerHostVO findOneByLongestRuntime() {
         return CollectionUtils.isNotEmpty(msHosts) ? msHosts.get(0) : null;
     }
 
+    @Override
+    public List<ManagementServerHostVO> listUpByIds(List<Long> ids) {
+        if (CollectionUtils.isEmpty(ids)) {
+            return new ArrayList<>();
+        }
+        SearchBuilder<ManagementServerHostVO> sb = createSearchBuilder();
+        sb.and("ids", sb.entity().getId(), SearchCriteria.Op.IN);
+        sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ManagementServerHostVO> sc = sb.create();
+        sc.setParameters("ids", ids.toArray());
+        sc.setParameters("state", ManagementServerHost.State.Up);
+        return listBy(sc);
+    }
 }