Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java
Original file line number Diff line number Diff line change
Expand Up @@ -1101,14 +1101,16 @@ public ExtensionJson toExtensionVersionJson(
}

json.setAllVersions(allVersionsJson);
var fileUrls = storageUtil.getFileUrls(
List.of(extVersion),
serverUrl,
withFileTypes(DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG, VSIXMANIFEST));
json.setFiles(fileUrls.get(extVersion.getId()));
if (json.getFiles().containsKey(DOWNLOAD_SIG)) {
json.getFiles().put(PUBLIC_KEY, UrlUtil.getPublicKeyUrl(extVersion));
}
// getFiles rather than getFileUrls: the package size is read from the same rows as the URLs, so
// this path stays one query, and toFilesJson is the same mapping the two builders below already
// use - including the public key it adds alongside a signature.
var resources = storageUtil
.getFiles(
List.of(extVersion),
withFileTypes(DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG, VSIXMANIFEST))
.get(extVersion.getId());
json.setFiles(toFilesJson(extVersion, resources, UrlUtil.createApiFileBaseUrl(serverUrl, extVersion)));
json.setDownloadSize(downloadSize(resources));
if (json.getDependencies() != null) {
for (var ref : json.getDependencies()) {
ref.setUrl(createApiUrl(serverUrl, "api", ref.getNamespace(), ref.getExtension()));
Expand Down Expand Up @@ -1189,6 +1191,7 @@ public ExtensionJson toExtensionVersionJson(
}

json.setFiles(files);
json.setDownloadSize(downloadSize(resources));
if (json.getDependencies() != null) {
for (var ref : json.getDependencies()) {
ref.setUrl(createApiUrl(serverUrl, "api", ref.getNamespace(), ref.getExtension()));
Expand Down Expand Up @@ -1238,6 +1241,7 @@ public ExtensionJson toExtensionVersionJsonV2(
json.getTargetPlatform(),
json.getVersion());
json.setFiles(toFilesJson(extVersion, resources, fileBaseUrl));
json.setDownloadSize(downloadSize(resources));
setExtensionReferenceUrls(json.getDependencies(), serverUrl);
setExtensionReferenceUrls(json.getBundledExtensions(), serverUrl);
return json;
Expand Down Expand Up @@ -1297,6 +1301,20 @@ private Map<String, String> toAllVersionsJson(
return allVersionsJson;
}

/**
* The size of the extension package itself, read from the {@code download} resource rather than summed
* over the version's files: readme, icon, changelog and license are stored as rows of their own but
* hold content that is already inside the package, so a sum would count it twice. Null where the
* package predates the size column and has not been backfilled yet.
*/
private Long downloadSize(List<FileResource> resources) {
return resources.stream()
.filter(resource -> DOWNLOAD.equals(resource.getType()))
.findFirst()
.map(FileResource::getSize)
.orElse(null);
}

private Map<String, String> toFilesJson(
ExtensionVersion extVersion,
List<FileResource> resources,
Expand Down
19 changes: 19 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/json/ExtensionJson.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ public static ExtensionJson error(String message) {
)
private Map<String, String> files;

@Schema(
description = "Size in bytes of the extension package - the file behind the 'download' entry of "
+ "'files' - for this version and target platform. Null where the size is not known, which "
+ "is the case for packages published before the registry started recording it and not yet "
+ "backfilled. This is the download size; the installed extension is larger, by whatever "
+ "the package's compression ratio happens to be."
)
private Long downloadSize;

@Schema(description = "Name of the extension")
@NotNull
private String name;
Expand Down Expand Up @@ -259,6 +268,14 @@ public void setFiles(Map<String, String> files) {
this.files = files;
}

public Long getDownloadSize() {
return downloadSize;
}

public void setDownloadSize(Long downloadSize) {
this.downloadSize = downloadSize;
}

public String getName() {
return name;
}
Expand Down Expand Up @@ -647,6 +664,7 @@ public boolean equals(Object o) {
return Objects.equals(namespaceUrl, that.namespaceUrl)
&& Objects.equals(reviewsUrl, that.reviewsUrl)
&& Objects.equals(files, that.files)
&& Objects.equals(downloadSize, that.downloadSize)
&& Objects.equals(name, that.name)
&& Objects.equals(namespace, that.namespace)
&& Objects.equals(targetPlatform, that.targetPlatform)
Expand Down Expand Up @@ -698,6 +716,7 @@ public int hashCode() {
namespaceUrl,
reviewsUrl,
files,
downloadSize,
name,
namespace,
targetPlatform,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,27 @@ public List<FileResource> findByType(Collection<ExtensionVersion> extVersions, C
}

var extVersionsById = extVersions.stream().collect(Collectors.toMap(ExtensionVersion::getId, ev -> ev));
return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE)
return dsl
.select(
FILE_RESOURCE.ID,
FILE_RESOURCE.EXTENSION_ID,
FILE_RESOURCE.NAME,
FILE_RESOURCE.TYPE,
FILE_RESOURCE.SIZE)
.from(FILE_RESOURCE)
.where(FILE_RESOURCE.EXTENSION_ID.in(extVersionsById.keySet())).and(FILE_RESOURCE.TYPE.in(types))
.fetch()
.map(row -> toFileResource(row, extVersionsById));
}

public List<FileResource> findAll(Collection<Long> extensionIds, Collection<String> types) {
return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE)
return dsl
.select(
FILE_RESOURCE.ID,
FILE_RESOURCE.EXTENSION_ID,
FILE_RESOURCE.NAME,
FILE_RESOURCE.TYPE,
FILE_RESOURCE.SIZE)
.from(FILE_RESOURCE)
.where(FILE_RESOURCE.EXTENSION_ID.in(extensionIds).and(FILE_RESOURCE.TYPE.in(types)))
.fetch()
Expand All @@ -76,6 +88,9 @@ private FileResource toFileResource(Record row, ExtensionVersion extVersion) {
fileResource.setId(row.get(FILE_RESOURCE.ID));
fileResource.setName(row.get(FILE_RESOURCE.NAME));
fileResource.setType(row.get(FILE_RESOURCE.TYPE));
// Null for a resource stored before V1_73 added the column that FileResourceSizeJobRequestHandler
// has not backfilled yet, so every reader has to treat the size as unknown rather than as zero.
fileResource.setSize(row.get(FILE_RESOURCE.SIZE));
fileResource.setExtension(extVersion);

return fileResource;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,28 @@ public long getFileSize(FileResource resource) throws IOException {
return storageService.getFileSize(resource);
}

/**
* The stored files of these versions, by version id, in one query.
* <p>
* {@link #getFileUrls} is the URL-only shorthand over this; a caller that needs more than the URL of a
* file - its size, say - takes the resources themselves rather than paying for a second query to read
* the other column.
*/
public Map<Long, List<FileResource>> getFiles(Collection<ExtensionVersion> extVersions, String... types) {
var byVersion = extVersions.stream()
.map(ev -> Map.<Long, List<FileResource>>entry(ev.getId(), new ArrayList<>(types.length)))
.collect(
Collectors.<Map.Entry<Long, List<FileResource>>, Long, List<FileResource>>toMap(
Map.Entry::getKey,
Map.Entry::getValue));

for (var resource : repositories.findFilesByType(extVersions, Arrays.asList(types))) {
byVersion.get(resource.getExtension().getId()).add(resource);
}

return byVersion;
}

/**
* Returns URLs for the given file types as a map of ExtensionVersion.id by a map of type by file URL, to be used in JSON response data.
*/
Expand All @@ -305,19 +327,17 @@ public Map<Long, Map<String, String>> getFileUrls(
String serverUrl,
String... types
) {
var type2Url = extVersions.stream()
.map(ev -> Map.<Long, Map<String, String>>entry(ev.getId(), new LinkedHashMap<>(types.length)))
.collect(
Collectors.<Map.Entry<Long, Map<String, String>>, Long, Map<String, String>>toMap(
Map.Entry::getKey,
Map.Entry::getValue));
var type2Url = HashMap.<Long, Map<String, String>>newHashMap(extVersions.size());
getFiles(extVersions, types).forEach((extVersionId, resources) -> {
var urls = new LinkedHashMap<String, String>(types.length);
for (var resource : resources) {
urls.put(
resource.getType(),
createApiFileUrl(serverUrl, resource.getExtension(), resource.getName()));
}

var resources = repositories.findFilesByType(extVersions, Arrays.asList(types));
for (var resource : resources) {
var extVersion = resource.getExtension();
type2Url.get(extVersion.getId())
.put(resource.getType(), createApiFileUrl(serverUrl, extVersion, resource.getName()));
}
type2Url.put(extVersionId, urls);
});

return type2Url;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ void shouldNotDeleteTempFileOnceOwnershipIsHandedToAsyncPublish() throws IOExcep
when(extensions.createExtensionFile(any())).thenReturn(tempFile);
when(tokens.useAccessToken(eq("tok"), any())).thenReturn(tau);
when(extensions.publishVersion(any(ExtensionProcessor.class), eq(tau))).thenReturn(extVersion);
when(storageUtilService.getFileUrls(any(), any(), any(String[].class))).thenReturn(Map.of(42L, Map.of()));
when(storageUtilService.getFiles(any(), any(String[].class))).thenReturn(Map.of(42L, List.of()));

registryService.publish(new ByteArrayInputStream(new byte[0]), "tok");

Expand Down
29 changes: 29 additions & 0 deletions server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ class RegistryAPITest {
*/
private static final Duration CHANGES_FEED_LAG = Duration.ofSeconds(30);

/** The size the mocked extension's .vsix reports, in bytes. */
private static final long DOWNLOAD_SIZE = 1_234_567L;

@MockitoSpyBean
UserService users;

Expand Down Expand Up @@ -220,6 +223,31 @@ void testExtension() throws Exception {
})));
}

// The size travels from file_resource.size to the JSON untouched, so that a client - the web UI's
// More Info box among them - can show how big the download is without fetching it.
@Test
void testExtensionDownloadSize() throws Exception {
var extVersion = mockExtension();
Mockito.when(repositories.findExtensionVersion("foo", "bar", null, VersionAlias.LATEST)).thenReturn(extVersion);

mockMvc.perform(get("/api/{namespace}/{extension}", "foo", "bar"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.downloadSize").value(DOWNLOAD_SIZE));
}

// Null for anything published before the column existed that the backfill has not reached yet.
// Omitted rather than sent as 0, which a client would have no way to tell from a genuinely empty file.
@Test
void testExtensionDownloadSizeUnknown() throws Exception {
var extVersion = mockExtension();
Mockito.when(repositories.findExtensionVersion("foo", "bar", null, VersionAlias.LATEST)).thenReturn(extVersion);
repositories.findFilesByType(List.of(extVersion), List.of(DOWNLOAD)).forEach(file -> file.setSize(null));

mockMvc.perform(get("/api/{namespace}/{extension}", "foo", "bar"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.downloadSize").doesNotExist());
}

@Test
void testExtensionWithPublicKey() throws Exception {
Mockito.when(integrityService.isEnabled()).thenReturn(true);
Expand Down Expand Up @@ -3022,6 +3050,7 @@ private ExtensionVersion mockExtension(String targetPlatform, boolean withSignat
download.setType(DOWNLOAD);
download.setStorageType(STORAGE_LOCAL);
download.setName("extension-1.0.0.vsix");
download.setSize(DOWNLOAD_SIZE);
var signature = new FileResource();
if (withSignature) {
signature.setExtension(extVersion);
Expand Down
24 changes: 6 additions & 18 deletions server/src/test/java/org/eclipse/openvsx/UserAPITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -416,12 +416,8 @@ void testGetOwnExtensionAsNamespaceOwner() throws Exception {
TargetPlatform.NAME_UNIVERSAL,
true,
false)))));
Mockito.when(
storageUtil.getFileUrls(
Mockito.anyCollection(),
Mockito.anyString(),
Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, new java.util.HashMap<>()));
Mockito.when(storageUtil.getFiles(Mockito.anyCollection(), Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, java.util.List.of()));
mockMvc.perform(
get("/user/extension/{namespace}/{extension}", "foobar", "baz")
.with(user("test_user")))
Expand Down Expand Up @@ -472,12 +468,8 @@ void testGetOwnExtensionAsNamespaceMember() throws Exception {
TargetPlatform.NAME_UNIVERSAL,
true,
false)))));
Mockito.when(
storageUtil.getFileUrls(
Mockito.anyCollection(),
Mockito.anyString(),
Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, new java.util.HashMap<>()));
Mockito.when(storageUtil.getFiles(Mockito.anyCollection(), Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, java.util.List.of()));
mockMvc.perform(
get("/user/extension/{namespace}/{extension}", "foobar", "baz")
.with(user("test_user")))
Expand All @@ -500,12 +492,8 @@ void testGetOwnExtensionNamespaceOwnershipConflict() throws Exception {
Mockito.when(repositories.findLatestVersion(eq("foobar"), eq("baz"), any(), eq(false), eq(false)))
.thenReturn(latest);
Mockito.when(repositories.findTargetPlatformsGroupedByVersion(extension)).thenReturn(List.of());
Mockito.when(
storageUtil.getFileUrls(
Mockito.anyCollection(),
Mockito.anyString(),
Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, new java.util.HashMap<>()));
Mockito.when(storageUtil.getFiles(Mockito.anyCollection(), Mockito.any(String[].class)))
.thenReturn(java.util.Map.of(42L, java.util.List.of()));

var scan = new ExtensionScan();
scan.setStatus(ScanStatus.QUARANTINED);
Expand Down
6 changes: 6 additions & 0 deletions webui/src/extension-registry-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ export interface Extension {
reviewsUrl: UrlString;
// key: file type, value: url
files: { [id: string]: UrlString };
/**
* Size in bytes of the .vsix behind `files.download`. Absent where the registry does not know it yet,
* which is the case for extensions published before it started recording sizes. This is the download
* size, not the size the extension takes up once installed.
*/
downloadSize?: number;

name: string;
namespace: string;
Expand Down
18 changes: 17 additions & 1 deletion webui/src/pages/extension-detail/extension-detail-overview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import GitHubIcon from '@mui/icons-material/GitHub';
import BugReportIcon from '@mui/icons-material/BugReport';
import QuestionAnswerIcon from '@mui/icons-material/QuestionAnswer';
import { MainContext } from '../../context';
import { addQuery, createRoute, getTargetPlatformDisplayName, getEngineDisplayName } from '../../utils';
import { addQuery, createRoute, formatFileSize, getTargetPlatformDisplayName, getEngineDisplayName } from '../../utils';
import { DelayedLoadIndicator } from '../../components/delayed-load-indicator';
import { SanitizedMarkdown } from '../../components/sanitized-markdown';
import { Timestamp } from '../../components/timestamp';
Expand Down Expand Up @@ -163,6 +163,19 @@ export const ExtensionDetailOverview: FunctionComponent<ExtensionDetailOverviewP
);
};

// The size of the .vsix, which is what the download costs - not what the extension takes up once
// installed, which is larger by however well the package happened to compress. Rendered only when the
// registry knows it: extensions published before it started recording sizes have none until the
// backfill reaches them, and a missing size is not a zero-byte one.
const renderSizeSection = (downloadSize: number): ReactNode => {
return (
<>
<Typography variant='h6'>Size</Typography>
<Typography variant='body2'>{formatFileSize(downloadSize)}</Typography>
</>
);
};

const renderAliasesSection = (otherAliases: string[], sx: SxProps<Theme>): ReactNode => {
const { extension } = props;
const aliasButtons = otherAliases.length
Expand Down Expand Up @@ -361,6 +374,9 @@ export const ExtensionDetailOverview: FunctionComponent<ExtensionDetailOverviewP

<Box sx={resourcesGroup}>
<Box>{renderIdentifierSection()}</Box>
{extension.downloadSize !== undefined ? (
<Box mt={2}>{renderSizeSection(extension.downloadSize)}</Box>
) : null}
</Box>

<Box sx={resourcesGroup}>
Expand Down
Loading
Loading