Skip to content

Commit 1ca2479

Browse files
author
amvanbaren
committed
Web extension resources return incorrect MIME type
Use [Content_Types].xml to set FileResource contentType Add test case for default content type: application/octet-stream Add migration to set FileResource.contentType Set contentType for RESOURCE type Prepend dot ('.') if extension doesn't start with a dot. Fix loggers Make migration run on every startup for testing purposes read inputstream before loading content types Get existing resource from db make uploadFile transactional move Transactional to MigrationService move transactional get file delete file use content of resource
1 parent f061c72 commit 1ca2479

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+950
-212
lines changed

‎server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java‎

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
********************************************************************************/
1010
package org.eclipse.openvsx;
1111

12+
import java.io.ByteArrayInputStream;
1213
import java.io.EOFException;
1314
import java.io.IOException;
1415
import java.nio.file.Path;
@@ -36,6 +37,11 @@
3637
import org.slf4j.Logger;
3738
import org.slf4j.LoggerFactory;
3839
import org.springframework.data.util.Pair;
40+
import org.springframework.http.MediaType;
41+
import org.xml.sax.SAXException;
42+
43+
import javax.xml.parsers.DocumentBuilderFactory;
44+
import javax.xml.parsers.ParserConfigurationException;
3945

4046
/**
4147
* Processes uploaded extension files and extracts their metadata.
@@ -282,17 +288,22 @@ private List<String> getEngines(JsonNode node) {
282288
}
283289

284290
public List<FileResource> getFileResources(ExtensionVersion extVersion) {
285-
var resources = new ArrayList<FileResource>();
291+
readInputStream();
292+
var contentTypes = loadContentTypes();
286293
var mappers = List.<Function<ExtensionVersion, FileResource>>of(
287294
this::getManifest, this::getReadme, this::getChangelog, this::getLicense, this::getIcon
288295
);
289296

290-
mappers.forEach(mapper -> Optional.of(extVersion).map(mapper).ifPresent(resources::add));
291-
return resources;
297+
return mappers.stream()
298+
.map(mapper -> mapper.apply(extVersion))
299+
.filter(Objects::nonNull)
300+
.map(resource -> setContentType(resource, contentTypes))
301+
.collect(Collectors.toList());
292302
}
293303

294304
public void processEachResource(ExtensionVersion extVersion, Consumer<FileResource> processor) {
295305
readInputStream();
306+
var contentTypes = loadContentTypes();
296307
zipFile.stream()
297308
.filter(zipEntry -> !zipEntry.isDirectory())
298309
.map(zipEntry -> {
@@ -311,6 +322,7 @@ public void processEachResource(ExtensionVersion extVersion, Consumer<FileResour
311322
resource.setName(zipEntry.getName());
312323
resource.setType(FileResource.RESOURCE);
313324
resource.setContent(bytes);
325+
setContentType(resource, contentTypes);
314326
return resource;
315327
})
316328
.filter(Objects::nonNull)
@@ -413,6 +425,46 @@ public FileResource getLicense(ExtensionVersion extVersion) {
413425
return license;
414426
}
415427

428+
private Map<String, String> loadContentTypes() {
429+
var bytes = ArchiveUtil.readEntry(zipFile, "[Content_Types].xml");
430+
var contentTypes = parseContentTypesXml(bytes);
431+
contentTypes.putIfAbsent(".vsix", "application/zip");
432+
return contentTypes;
433+
}
434+
435+
private Map<String, String> parseContentTypesXml(byte[] content) {
436+
var contentTypes = new HashMap<String, String>();
437+
try (var input = new ByteArrayInputStream(content)) {
438+
var document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input);
439+
var elements = document.getDocumentElement().getElementsByTagName("Default");
440+
for(var i = 0; i < elements.getLength(); i++) {
441+
var element = elements.item(i);
442+
var attributes = element.getAttributes();
443+
var extension = attributes.getNamedItem("Extension").getTextContent();
444+
if(!extension.startsWith(".")) {
445+
extension = "." + extension;
446+
}
447+
448+
var contentType = attributes.getNamedItem("ContentType").getTextContent();
449+
contentTypes.put(extension, contentType);
450+
}
451+
} catch (IOException | ParserConfigurationException | SAXException e) {
452+
logger.error("failed to read content types", e);
453+
contentTypes.clear();
454+
}
455+
456+
return contentTypes;
457+
}
458+
459+
private FileResource setContentType(FileResource resource, Map<String, String> contentTypes) {
460+
var resourceName = Optional.ofNullable(resource.getName()).orElse("");
461+
var fileExtensionIndex = resourceName.lastIndexOf('.');
462+
var fileExtension = fileExtensionIndex != -1 ? resourceName.substring(fileExtensionIndex) : "";
463+
var contentType = contentTypes.getOrDefault(fileExtension, MediaType.APPLICATION_OCTET_STREAM_VALUE);
464+
resource.setContentType(contentType);
465+
return resource;
466+
}
467+
416468
private void detectLicense(byte[] content, ExtensionVersion extVersion) {
417469
if (Strings.isNullOrEmpty(extVersion.getLicense())) {
418470
var detection = new LicenseDetection();

‎server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,7 @@ private ResponseEntity<byte[]> browseFile(
404404
String version
405405
) {
406406
if (resource.getStorageType().equals(FileResource.STORAGE_DB)) {
407-
var headers = storageUtil.getFileResponseHeaders(resource.getName());
407+
var headers = storageUtil.getFileResponseHeaders(resource);
408408
return new ResponseEntity<>(resource.getContent(), headers, HttpStatus.OK);
409409
} else {
410410
var namespace = new Namespace();

‎server/src/main/java/org/eclipse/openvsx/entities/FileResource.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ public class FileResource {
4545
@Basic(fetch = FetchType.LAZY)
4646
byte[] content;
4747

48+
String contentType;
49+
4850
@Column(length = 32)
4951
String storageType;
5052

@@ -88,6 +90,14 @@ public void setContent(byte[] content) {
8890
this.content = content;
8991
}
9092

93+
public String getContentType() {
94+
return contentType;
95+
}
96+
97+
public void setContentType(String contentType) {
98+
this.contentType = contentType;
99+
}
100+
91101
public String getStorageType() {
92102
return storageType;
93103
}

‎server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
package org.eclipse.openvsx.migration;
1111

1212
import org.eclipse.openvsx.ExtensionProcessor;
13+
import org.eclipse.openvsx.entities.ExtensionVersion;
1314
import org.jobrunr.jobs.annotations.Job;
14-
import org.jobrunr.jobs.context.JobRunrDashboardLogger;
1515
import org.jobrunr.jobs.lambdas.JobRequestHandler;
1616
import org.slf4j.Logger;
1717
import org.slf4j.LoggerFactory;
@@ -21,7 +21,7 @@
2121
@Component
2222
public class ExtractResourcesJobRequestHandler implements JobRequestHandler<MigrationJobRequest> {
2323

24-
protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class));
24+
protected final Logger logger = LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class);
2525

2626
@Autowired
2727
ExtractResourcesJobService service;
@@ -32,12 +32,12 @@ public class ExtractResourcesJobRequestHandler implements JobRequestHandler<Migr
3232
@Override
3333
@Job(name = "Extract resources from published extension version", retries = 3)
3434
public void run(MigrationJobRequest jobRequest) throws Exception {
35-
var extVersion = service.getExtension(jobRequest.getEntityId());
35+
var extVersion = migrations.find(jobRequest, ExtensionVersion.class);
3636
logger.info("Extracting resources for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform());
3737

3838
service.deleteResources(extVersion);
39-
var entry = service.getDownload(extVersion);
40-
var extensionFile = migrations.getExtensionFile(entry);
39+
var entry = migrations.getDownload(extVersion);
40+
var extensionFile = migrations.getFile(entry);
4141
var download = entry.getKey();
4242
try(var extProcessor = new ExtensionProcessor(extensionFile)) {
4343
extProcessor.processEachResource(download.getExtension(), (resource) -> {
@@ -48,5 +48,6 @@ public void run(MigrationJobRequest jobRequest) throws Exception {
4848
}
4949

5050
service.deleteWebResources(extVersion);
51+
migrations.deleteFile(extensionFile);
5152
}
5253
}

‎server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,11 @@
1212
import org.eclipse.openvsx.entities.ExtensionVersion;
1313
import org.eclipse.openvsx.entities.FileResource;
1414
import org.eclipse.openvsx.repositories.RepositoryService;
15-
import org.eclipse.openvsx.storage.AzureBlobStorageService;
16-
import org.eclipse.openvsx.storage.GoogleCloudStorageService;
17-
import org.eclipse.openvsx.storage.IStorageService;
1815
import org.springframework.beans.factory.annotation.Autowired;
19-
import org.springframework.http.HttpMethod;
20-
import org.springframework.retry.annotation.Retryable;
2116
import org.springframework.stereotype.Component;
22-
import org.springframework.web.client.RestTemplate;
2317

2418
import javax.persistence.EntityManager;
2519
import javax.transaction.Transactional;
26-
import java.io.IOException;
27-
import java.nio.file.Files;
28-
import java.nio.file.Path;
2920
import java.util.AbstractMap;
3021
import java.util.Map;
3122

@@ -35,22 +26,9 @@ public class ExtractResourcesJobService {
3526
@Autowired
3627
RepositoryService repositories;
3728

38-
@Autowired
39-
RestTemplate restTemplate;
40-
4129
@Autowired
4230
EntityManager entityManager;
4331

44-
@Autowired
45-
AzureBlobStorageService azureStorage;
46-
47-
@Autowired
48-
GoogleCloudStorageService googleStorage;
49-
50-
public ExtensionVersion getExtension(long entityId) {
51-
return entityManager.find(ExtensionVersion.class, entityId);
52-
}
53-
5432
@Transactional
5533
public void deleteResources(ExtensionVersion extVersion) {
5634
repositories.deleteFileResources(extVersion, "resource");
@@ -61,13 +39,6 @@ public void deleteWebResources(ExtensionVersion extVersion) {
6139
repositories.deleteFileResources(extVersion, "web-resource");
6240
}
6341

64-
@Transactional
65-
public Map.Entry<FileResource, byte[]> getDownload(ExtensionVersion extVersion) {
66-
var download = repositories.findFileByType(extVersion, FileResource.DOWNLOAD);
67-
var content = download.getStorageType().equals(FileResource.STORAGE_DB) ? download.getContent() : null;
68-
return new AbstractMap.SimpleEntry<>(download, content);
69-
}
70-
7142
@Transactional
7243
public void persistResource(FileResource resource) {
7344
entityManager.persist(resource);

‎server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
import javax.transaction.Transactional;
2222
import java.nio.charset.StandardCharsets;
23+
import java.time.LocalDateTime;
24+
import java.time.ZoneOffset;
2325
import java.util.UUID;
2426

2527
@Component
@@ -41,6 +43,8 @@ public void runMigrations(ApplicationStartedEvent event) {
4143
extractResourcesMigration();
4244
setPreReleaseMigration();
4345
renameDownloadsMigration();
46+
resetContentTypeMigration(); // TODO remove when done debugging
47+
setContentTypeMigration();
4448
}
4549

4650
private void extractResourcesMigration() {
@@ -61,6 +65,17 @@ private void renameDownloadsMigration() {
6165
repositories.findNotMigratedRenamedDownloads().forEach(item -> enqueueJob(jobName, handler, item));
6266
}
6367

68+
// TODO remove when done debugging
69+
private void resetContentTypeMigration() {
70+
repositories.findMigratedContentTypes().forEach(item -> item.setMigrationScheduled(false));
71+
}
72+
73+
private void setContentTypeMigration() {
74+
var jobName = "SetContentTypeMigration" + LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); // TODO remove when done debugging
75+
var handler = SetContentTypeJobRequestHandler.class;
76+
repositories.findNotMigratedContentTypes().forEach(item -> enqueueJob(jobName, handler, item));
77+
}
78+
6479
private void enqueueJob(String jobName, Class<? extends JobRequestHandler<MigrationJobRequest>> handler, MigrationItem item) {
6580
var jobIdText = jobName + "::itemId=" + item.getId();
6681
var jobId = UUID.nameUUIDFromBytes(jobIdText.getBytes(StandardCharsets.UTF_8));

‎server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
* ****************************************************************************** */
1010
package org.eclipse.openvsx.migration;
1111

12+
import org.eclipse.openvsx.entities.ExtensionVersion;
1213
import org.eclipse.openvsx.entities.FileResource;
14+
import org.eclipse.openvsx.repositories.RepositoryService;
1315
import org.eclipse.openvsx.storage.AzureBlobStorageService;
1416
import org.eclipse.openvsx.storage.GoogleCloudStorageService;
1517
import org.eclipse.openvsx.storage.IStorageService;
@@ -19,14 +21,23 @@
1921
import org.springframework.stereotype.Component;
2022
import org.springframework.web.client.RestTemplate;
2123

24+
import javax.persistence.EntityManager;
25+
import javax.transaction.Transactional;
2226
import java.io.IOException;
2327
import java.nio.file.Files;
2428
import java.nio.file.Path;
29+
import java.util.AbstractMap;
2530
import java.util.Map;
2631

2732
@Component
2833
public class MigrationService {
2934

35+
@Autowired
36+
EntityManager entityManager;
37+
38+
@Autowired
39+
RepositoryService repositories;
40+
3041
@Autowired
3142
RestTemplate restTemplate;
3243

@@ -36,20 +47,38 @@ public class MigrationService {
3647
@Autowired
3748
GoogleCloudStorageService googleStorage;
3849

50+
public <T> T find(MigrationJobRequest jobRequest, Class<T> clazz) {
51+
return entityManager.find(clazz, jobRequest.getEntityId());
52+
}
53+
54+
@Transactional
55+
public Map.Entry<FileResource, byte[]> getDownload(ExtensionVersion extVersion) {
56+
var download = repositories.findFileByType(extVersion, FileResource.DOWNLOAD);
57+
var content = download.getStorageType().equals(FileResource.STORAGE_DB) ? download.getContent() : null;
58+
return new AbstractMap.SimpleEntry<>(download, content);
59+
}
60+
61+
@Transactional
62+
public byte[] getContent(FileResource download) {
63+
download = entityManager.merge(download);
64+
return download.getStorageType().equals(FileResource.STORAGE_DB) ? download.getContent() : null;
65+
}
66+
3967
@Retryable
40-
public Path getExtensionFile(Map.Entry<FileResource, byte[]> entry) {
68+
public Path getFile(Map.Entry<FileResource, byte[]> entry) {
4169
Path extensionFile;
70+
var resource = entry.getKey();
4271
try {
43-
extensionFile = Files.createTempFile("extension_", ".vsix");
72+
var suffixIndex = resource.getName().lastIndexOf('.');
73+
extensionFile = Files.createTempFile("file_", resource.getName().substring(suffixIndex));
4474
} catch (IOException e) {
45-
throw new RuntimeException("Failed to create extension file", e);
75+
throw new RuntimeException("Failed to create file", e);
4676
}
4777

4878
var content = entry.getValue();
4979
if(content == null) {
50-
var download = entry.getKey();
51-
var storage = getStorage(download);
52-
var uri = storage.getLocation(download);
80+
var storage = getStorage(resource);
81+
var uri = storage.getLocation(resource);
5382
restTemplate.execute(uri, HttpMethod.GET, null, response -> {
5483
try(var out = Files.newOutputStream(extensionFile)) {
5584
response.getBody().transferTo(out);
@@ -61,19 +90,29 @@ public Path getExtensionFile(Map.Entry<FileResource, byte[]> entry) {
6190
try {
6291
Files.write(extensionFile, content);
6392
} catch (IOException e) {
64-
throw new RuntimeException("Failed to write to extension file", e);
93+
throw new RuntimeException("Failed to write to file", e);
6594
}
6695
}
6796

6897
return extensionFile;
6998
}
7099

100+
public void deleteFile(Path filePath) {
101+
try {
102+
Files.delete(filePath);
103+
} catch (IOException e) {
104+
throw new RuntimeException("Failed to delete file");
105+
}
106+
}
107+
71108
@Retryable
109+
@Transactional
72110
public void uploadResource(FileResource resource) {
73111
if(resource.getStorageType().equals(FileResource.STORAGE_DB)) {
74112
return;
75113
}
76114

115+
resource = entityManager.merge(resource);
77116
var storage = getStorage(resource);
78117
storage.uploadFile(resource);
79118
resource.setContent(null);

0 commit comments

Comments
 (0)