Skip to content

Commit dc4d7e8

Browse files
author
amvanbaren
committed
Cache /api/-/search results
1 parent a874a24 commit dc4d7e8

File tree

11 files changed

+225
-81
lines changed

11 files changed

+225
-81
lines changed

‎server/src/main/java/org/eclipse/openvsx/ExtensionService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ public void updateExtension(Extension extension) {
117117
cache.evictNamespaceDetails(extension);
118118
cache.evictLatestExtensionVersion(extension);
119119
cache.evictExtensionJsons(extension);
120+
cache.evictSearchEntryJsons(extension);
120121

121122
if (extension.getVersions().stream().anyMatch(ExtensionVersion::isActive)) {
122123
// There is at least one active version => activate the extension

‎server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java‎

Lines changed: 19 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.eclipse.openvsx.entities.*;
3030
import org.eclipse.openvsx.json.*;
3131
import org.eclipse.openvsx.repositories.RepositoryService;
32-
import org.eclipse.openvsx.search.ExtensionSearch;
3332
import org.eclipse.openvsx.search.ISearchService;
3433
import org.eclipse.openvsx.search.SearchUtilService;
3534
import org.eclipse.openvsx.storage.StorageUtilService;
@@ -39,8 +38,6 @@
3938
import org.springframework.beans.factory.annotation.Autowired;
4039
import org.springframework.cache.annotation.Cacheable;
4140
import org.springframework.dao.DataIntegrityViolationException;
42-
import org.springframework.data.elasticsearch.core.SearchHit;
43-
import org.springframework.data.elasticsearch.core.SearchHits;
4441
import org.springframework.http.HttpStatus;
4542
import org.springframework.http.ResponseEntity;
4643
import org.springframework.retry.annotation.Retryable;
@@ -82,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry {
8279
@Autowired
8380
CacheService cache;
8481

82+
@Autowired
83+
SearchEntryService searchEntries;
84+
8585
@Override
8686
public NamespaceJson getNamespace(String namespaceName) {
8787
var namespace = repositories.findNamespace(namespaceName);
@@ -211,7 +211,22 @@ public SearchResultJson search(ISearchService.Options options) {
211211
}
212212

213213
var searchHits = search.search(options);
214-
json.extensions = toSearchEntries(searchHits, options);
214+
var extensions = new ArrayList<SearchEntryJson>();
215+
for (var searchHit : searchHits) {
216+
var searchEntry = searchEntries.toJson(searchHit, options.includeAllVersions);
217+
if(searchEntry != null) {
218+
// use averageRating, reviewCount and downloadCount from ElasticSearch response,
219+
// so that cached SearchEntryJson doesn't have to be evicted every time
220+
// averageRating, reviewCount or downloadCount are updated.
221+
var extensionSearch = searchHit.getContent();
222+
searchEntry.averageRating = extensionSearch.averageRating;
223+
searchEntry.reviewCount = extensionSearch.reviewCount;
224+
searchEntry.downloadCount = extensionSearch.downloadCount;
225+
extensions.add(searchEntry);
226+
}
227+
}
228+
229+
json.extensions = extensions;
215230
json.offset = options.requestedOffset;
216231
json.totalSize = (int) searchHits.getTotalHits();
217232
return json;
@@ -741,81 +756,6 @@ public ResultJson deleteReview(String namespace, String extensionName) {
741756
return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName());
742757
}
743758

744-
private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
745-
var searchItem = searchHit.getContent();
746-
var extension = entityManager.find(Extension.class, searchItem.id);
747-
if (extension == null || !extension.isActive()) {
748-
extension = new Extension();
749-
extension.setId(searchItem.id);
750-
search.removeSearchEntry(extension);
751-
return null;
752-
}
753-
754-
return extension;
755-
}
756-
757-
private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
758-
var serverUrl = UrlUtil.getBaseUrl();
759-
var extensions = searchHits.stream()
760-
.map(this::getExtension)
761-
.filter(Objects::nonNull)
762-
.collect(Collectors.toList());
763-
764-
var latestVersions = extensions.stream()
765-
.map(e -> {
766-
var latest = versions.getLatestTrxn(e, null, false, true);
767-
return new AbstractMap.SimpleEntry<>(e.getId(), latest);
768-
})
769-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
770-
771-
var searchEntries = latestVersions.entrySet().stream()
772-
.map(e -> {
773-
var entry = e.getValue().toSearchEntryJson();
774-
entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name);
775-
return new AbstractMap.SimpleEntry<>(e.getKey(), entry);
776-
})
777-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
778-
779-
var fileUrls = storageUtil.getFileUrls(latestVersions.values(), serverUrl, DOWNLOAD, ICON);
780-
searchEntries.forEach((extensionId, searchEntry) -> searchEntry.files = fileUrls.get(latestVersions.get(extensionId).getId()));
781-
if (options.includeAllVersions) {
782-
var allActiveVersions = repositories.findActiveVersions(extensions).stream()
783-
.sorted(ExtensionVersion.SORT_COMPARATOR)
784-
.collect(Collectors.toList());
785-
786-
var activeVersionsByExtensionId = allActiveVersions.stream()
787-
.collect(Collectors.groupingBy(ev -> ev.getExtension().getId()));
788-
789-
var versionUrls = storageUtil.getFileUrls(allActiveVersions, serverUrl, DOWNLOAD);
790-
for(var extension : extensions) {
791-
var activeVersions = activeVersionsByExtensionId.get(extension.getId());
792-
var searchEntry = searchEntries.get(extension.getId());
793-
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
794-
}
795-
}
796-
797-
return extensions.stream()
798-
.map(Extension::getId)
799-
.map(searchEntries::get)
800-
.collect(Collectors.toList());
801-
}
802-
803-
private List<SearchEntryJson.VersionReference> getAllVersionReferences(
804-
List<ExtensionVersion> extVersions,
805-
Map<Long, Map<String, String>> versionUrls,
806-
String serverUrl
807-
) {
808-
Collections.sort(extVersions, ExtensionVersion.SORT_COMPARATOR);
809-
return extVersions.stream().map(extVersion -> {
810-
var ref = new SearchEntryJson.VersionReference();
811-
ref.version = extVersion.getVersion();
812-
ref.engines = extVersion.getEnginesMap();
813-
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
814-
ref.files = versionUrls.get(extVersion.getId());
815-
return ref;
816-
}).collect(Collectors.toList());
817-
}
818-
819759
public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String targetPlatform, boolean onlyActive, boolean inTransaction) {
820760
var extension = extVersion.getExtension();
821761
var latest = inTransaction

‎server/src/main/java/org/eclipse/openvsx/cache/CacheService.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@
1212
import org.eclipse.openvsx.entities.Extension;
1313
import org.eclipse.openvsx.entities.ExtensionVersion;
1414
import org.eclipse.openvsx.entities.UserData;
15+
import org.eclipse.openvsx.json.SearchEntryJson;
1516
import org.eclipse.openvsx.repositories.RepositoryService;
17+
import org.eclipse.openvsx.search.ExtensionSearch;
1618
import org.eclipse.openvsx.util.TargetPlatform;
1719
import org.eclipse.openvsx.util.VersionAlias;
1820
import org.springframework.beans.factory.annotation.Autowired;
1921
import org.springframework.cache.CacheManager;
22+
import org.springframework.data.elasticsearch.core.SearchHit;
2023
import org.springframework.stereotype.Component;
2124

2225
import java.util.ArrayList;
@@ -27,6 +30,7 @@ public class CacheService {
2730

2831
public static final String CACHE_DATABASE_SEARCH = "database.search";
2932
public static final String CACHE_EXTENSION_JSON = "extension.json";
33+
public static final String CACHE_SEARCH_ENTRY_JSON = "search.entry.json";
3034
public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
3135
public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
3236
public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating";
@@ -43,6 +47,9 @@ public class CacheService {
4347
@Autowired
4448
ExtensionJsonCacheKeyGenerator extensionJsonCacheKey;
4549

50+
@Autowired
51+
SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator;
52+
4653
@Autowired
4754
LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKey;
4855

@@ -92,6 +99,33 @@ public void evictExtensionJsons(Extension extension) {
9299
}
93100
}
94101

102+
public SearchEntryJson getSearchEntryJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
103+
var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
104+
return cache != null
105+
? cache.get(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), SearchEntryJson.class)
106+
: null;
107+
}
108+
109+
public void putSearchEntryJson(SearchEntryJson searchEntry, SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
110+
var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
111+
if(cache != null) {
112+
cache.put(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), searchEntry);
113+
}
114+
}
115+
116+
public void evictSearchEntryJsons(Extension extension) {
117+
var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
118+
if(cache == null) {
119+
return; // cache is not created
120+
}
121+
122+
var includeAllVersionsList = List.of(true, false);
123+
for(var includeAllVersions : includeAllVersionsList) {
124+
var key = searchEntryJsonCacheKeyGenerator.generate(extension.getId(), includeAllVersions);
125+
cache.evictIfPresent(key);
126+
}
127+
}
128+
95129
public void evictLatestExtensionVersion(Extension extension) {
96130
var cache = cacheManager.getCache(CACHE_LATEST_EXTENSION_VERSION);
97131
if(cache != null) {

‎server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.cache;
11+
12+
import org.springframework.stereotype.Component;
13+
14+
@Component
15+
public class SearchEntryJsonCacheKeyGenerator {
16+
17+
public Object generate(long extensionId, boolean includeAllVersions) {
18+
return "extensionId=" + extensionId + ",includeAllVersions=" + includeAllVersions;
19+
}
20+
}

‎server/src/main/java/org/eclipse/openvsx/entities/Extension.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public ExtensionSearch toSearch(ExtensionVersion latest) {
6262
search.name = this.getName();
6363
search.namespace = this.getNamespace().getName();
6464
search.extensionId = search.namespace + "." + search.name;
65+
search.averageRating = this.getAverageRating();
6566
search.downloadCount = this.getDownloadCount();
6667
search.targetPlatforms = this.getVersions().stream()
6768
.map(ExtensionVersion::getTargetPlatform)

‎server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public class SearchEntryJson implements Serializable {
7878
name = "VersionReference",
7979
description = "Essential metadata of an extension version"
8080
)
81-
public static class VersionReference {
81+
public static class VersionReference implements Serializable {
8282

8383
@Schema(description = "URL to get the full metadata of this version")
8484
public String url;

‎server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.json;
11+
12+
import org.eclipse.openvsx.cache.CacheService;
13+
import org.eclipse.openvsx.entities.Extension;
14+
import org.eclipse.openvsx.entities.ExtensionVersion;
15+
import org.eclipse.openvsx.repositories.RepositoryService;
16+
import org.eclipse.openvsx.search.ExtensionSearch;
17+
import org.eclipse.openvsx.search.SearchUtilService;
18+
import org.eclipse.openvsx.storage.StorageUtilService;
19+
import org.eclipse.openvsx.util.UrlUtil;
20+
import org.eclipse.openvsx.util.VersionService;
21+
import org.springframework.beans.factory.annotation.Autowired;
22+
import org.springframework.data.elasticsearch.core.SearchHit;
23+
import org.springframework.stereotype.Component;
24+
25+
import javax.persistence.EntityManager;
26+
import javax.transaction.Transactional;
27+
import java.util.List;
28+
import java.util.Map;
29+
import java.util.stream.Collectors;
30+
31+
import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD;
32+
import static org.eclipse.openvsx.entities.FileResource.ICON;
33+
import static org.eclipse.openvsx.util.UrlUtil.createApiUrl;
34+
35+
@Component
36+
public class SearchEntryService {
37+
38+
@Autowired
39+
EntityManager entityManager;
40+
41+
@Autowired
42+
VersionService versions;
43+
44+
@Autowired
45+
StorageUtilService storageUtil;
46+
47+
@Autowired
48+
SearchUtilService search;
49+
50+
@Autowired
51+
RepositoryService repositories;
52+
53+
@Autowired
54+
CacheService cache;
55+
56+
@Transactional
57+
public SearchEntryJson toJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
58+
var searchEntry = cache.getSearchEntryJson(searchHit, includeAllVersions);
59+
if(searchEntry != null) {
60+
return searchEntry;
61+
}
62+
63+
var extension = getExtension(searchHit);
64+
if(extension == null) {
65+
return null;
66+
}
67+
68+
var serverUrl = UrlUtil.getBaseUrl();
69+
if(includeAllVersions && cache != null) {
70+
searchEntry = cache.getSearchEntryJson(searchHit, false);
71+
}
72+
if(searchEntry == null) {
73+
var latest = versions.getLatest(extension, null, false, true);
74+
searchEntry = latest.toSearchEntryJson();
75+
searchEntry.url = createApiUrl(serverUrl, "api", searchEntry.namespace, searchEntry.name);
76+
searchEntry.files = storageUtil.getFileUrls(latest, serverUrl, DOWNLOAD, ICON);
77+
cache.putSearchEntryJson(searchEntry, searchHit, false);
78+
}
79+
if (includeAllVersions) {
80+
var activeVersions = repositories.findActiveVersions(extension).toList();
81+
var versionUrls = storageUtil.getFileUrls(activeVersions, serverUrl, DOWNLOAD);
82+
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
83+
cache.putSearchEntryJson(searchEntry, searchHit, true);
84+
}
85+
86+
return searchEntry;
87+
}
88+
89+
private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
90+
var searchItem = searchHit.getContent();
91+
var extension = entityManager.find(Extension.class, searchItem.id);
92+
if (extension == null || !extension.isActive()) {
93+
extension = new Extension();
94+
extension.setId(searchItem.id);
95+
search.removeSearchEntry(extension);
96+
return null;
97+
}
98+
99+
return extension;
100+
}
101+
102+
private List<SearchEntryJson.VersionReference> getAllVersionReferences(
103+
List<ExtensionVersion> extVersions,
104+
Map<Long, Map<String, String>> versionUrls,
105+
String serverUrl
106+
) {
107+
return extVersions.stream()
108+
.sorted(ExtensionVersion.SORT_COMPARATOR)
109+
.map(extVersion -> {
110+
var ref = new SearchEntryJson.VersionReference();
111+
ref.version = extVersion.getVersion();
112+
ref.engines = extVersion.getEnginesMap();
113+
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
114+
ref.files = versionUrls.get(extVersion.getId());
115+
return ref;
116+
}).collect(Collectors.toList());
117+
}
118+
}

‎server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ public class ExtensionSearch implements Serializable {
4444
@Field(index = false)
4545
public long timestamp;
4646

47+
@Nullable
48+
@Field(index = false, type = FieldType.Float)
49+
public Double averageRating;
50+
51+
@Nullable
52+
@Field(index = false, type = FieldType.Float)
53+
public Long reviewCount;
54+
4755
@Nullable
4856
@Field(index = false, type = FieldType.Float)
4957
public Double rating;

‎server/src/main/resources/ehcache.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@
3030
<heap unit="entries">1024</heap>
3131
</resources>
3232
</cache>
33+
<cache alias="search.entry.json">
34+
<expiry>
35+
<ttl unit="seconds">3600</ttl>
36+
</expiry>
37+
<resources>
38+
<heap unit="entries">1024</heap>
39+
<offheap unit="MB">32</offheap>
40+
<disk unit="MB">128</disk>
41+
</resources>
42+
</cache>
3343
<cache alias="extension.json">
3444
<expiry>
3545
<ttl unit="seconds">3600</ttl>

0 commit comments

Comments
 (0)