文件上传增加S3协议的OSS支持

hsweb-system/hsweb-system-file/pom.xml

@@ -54,6 +54,13 @@
             <artifactId>spring-test</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+
     </dependencies>
 
 </project>

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/FileServiceConfiguration.java

@@ -2,35 +2,66 @@
 
 import org.hswebframework.web.file.service.FileStorageService;
 import org.hswebframework.web.file.service.LocalFileStorageService;
+import org.hswebframework.web.file.service.S3FileStorageService;
 import org.hswebframework.web.file.web.ReactiveFileController;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.net.URI;
 
 @AutoConfiguration
-@EnableConfigurationProperties(FileUploadProperties.class)
+@EnableConfigurationProperties({FileUploadProperties.class,S3StorageProperties.class})
 public class FileServiceConfiguration {
 
 
     @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.REACTIVE)
     static class ReactiveConfiguration {
 
         @Bean
-        @ConditionalOnMissingBean(FileStorageService.class)
+        @ConditionalOnProperty(name = "file.storage", havingValue = "local", matchIfMissing = true)
         public FileStorageService fileStorageService(FileUploadProperties properties) {
             return new LocalFileStorageService(properties);
         }
 
         @Bean
-        @ConditionalOnMissingBean(name = "reactiveFileController")
+        @ConditionalOnProperty(name = "file.storage", havingValue = "s3")
         public ReactiveFileController reactiveFileController(FileUploadProperties properties,
                                                              FileStorageService storageService) {
             return new ReactiveFileController(properties, storageService);
         }
 
+
+
+
+        @Bean
+        @ConditionalOnMissingBean(FileStorageService.class)
+        public FileStorageService s3FileStorageService(S3StorageProperties properties,
+                                                         S3Client s3Client) {
+            return new S3FileStorageService(properties, s3Client);
+        }
+
+        @Bean
+        @ConditionalOnProperty(name = "file.storage", havingValue = "s3")
+        public S3Client s3Client(S3StorageProperties properties) {
+            return S3Client.builder()
+                    .endpointOverride(URI.create(properties.getEndpoint()))
+                    .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(
+                            properties.getAccessKey(), properties.getSecretKey()
+                    )))
+                    .region(Region.of(properties.getRegion()))
+                    .build();
+        }
+
     }
 
 }

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/S3StorageProperties.java

@@ -0,0 +1,85 @@
+package org.hswebframework.web.file;
+
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.http.MediaType;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.Locale;
+import java.util.Set;
+
+@Data
+@ConfigurationProperties(prefix = "oss.s3")
+public class S3StorageProperties {
+    private String endpoint;
+    private String region;
+    private String accessKey;
+    private String secretKey;
+    private String bucket;
+    private String baseUrl;
+
+    private Set<String> allowFiles;
+
+    private Set<String> denyFiles;
+
+    private Set<String> allowMediaType;
+
+    private Set<String> denyMediaType;
+
+    private Set<PosixFilePermission> permissions;
+
+
+    public void applyFilePermission(File file) {
+
+        if (CollectionUtils.isEmpty(permissions)) {
+            return;
+        }
+        try {
+            Path path = Paths.get(file.toURI());
+            PosixFileAttributeView view = Files.getFileAttributeView(path, PosixFileAttributeView.class);
+            view.setPermissions(permissions);
+        } catch (Throwable ignore) {
+            // 失败时忽略，兼容Windows等不支持Posix的系统
+        }
+    }
+
+    public boolean denied(String name, MediaType mediaType) {
+        String suffix = (name.contains(".") ? name.substring(name.lastIndexOf(".") + 1) : "").toLowerCase(Locale.ROOT);
+        boolean defaultDeny = false;
+        if (CollectionUtils.isNotEmpty(denyFiles)) {
+            if (denyFiles.contains(suffix)) {
+                return true;
+            }
+            defaultDeny = false;
+        }
+
+        if (CollectionUtils.isNotEmpty(allowFiles)) {
+            if (allowFiles.contains(suffix)) {
+                return false;
+            }
+            defaultDeny = true;
+        }
+
+        if (CollectionUtils.isNotEmpty(denyMediaType)) {
+            if (denyMediaType.contains(mediaType.toString())) {
+                return true;
+            }
+            defaultDeny = false;
+        }
+
+        if (CollectionUtils.isNotEmpty(allowMediaType)) {
+            if (allowMediaType.contains(mediaType.toString())) {
+                return false;
+            }
+            defaultDeny = true;
+        }
+
+        return defaultDeny;
+    }
+}

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/service/FileStorageService.java

@@ -3,6 +3,7 @@
 import org.springframework.http.codec.multipart.FilePart;
 import reactor.core.publisher.Mono;
 
+import java.io.IOException;
 import java.io.InputStream;
 
 /**
@@ -19,7 +20,7 @@ public interface FileStorageService {
      * @param filePart FilePart
      * @return 文件访问地址
      */
-    Mono<String> saveFile(FilePart filePart);
+    Mono<String> saveFile(FilePart filePart) throws IOException;
 
     /**
      * 使用文件流保存文件,并返回文件地址

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/service/S3FileStorageService.java

@@ -0,0 +1,86 @@
+package org.hswebframework.web.file.service;
+
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import org.hswebframework.web.file.S3StorageProperties;
+import org.springframework.core.io.buffer.DataBuffer;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.http.MediaType;
+import org.springframework.http.codec.multipart.FilePart;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+import software.amazon.awssdk.core.async.AsyncRequestBody;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Locale;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@AllArgsConstructor
+public class S3FileStorageService implements FileStorageService {
+
+    private final S3StorageProperties properties;
+
+    private final S3Client s3Client;
+
+    @Override
+    public Mono<String> saveFile(FilePart filePart) {
+        String filename = buildFileName(filePart.filename());
+        return DataBufferUtils.join(filePart.content())
+                .publishOn(Schedulers.boundedElastic())
+                .map(dataBuffer -> {
+                    byte[] bytes = new byte[dataBuffer.readableByteCount()];
+                    dataBuffer.read(bytes);
+                    DataBufferUtils.release(dataBuffer);
+                    return new ByteArrayInputStream(bytes);
+                })
+                .map(inputStream -> {
+                    PutObjectRequest request = PutObjectRequest.builder()
+                            .bucket(properties.getBucket())
+                            .key(filename)
+                            .build();
+
+                    s3Client.putObject(request, RequestBody.fromInputStream(inputStream, inputStream.available()));
+                    return buildFileUrl(filename);
+                });
+    }
+
+
+    @Override
+    @SneakyThrows
+    public Mono<String> saveFile(InputStream inputStream, String fileType) {
+        return Mono.fromCallable(() -> {
+                    String key = UUID.randomUUID().toString() + (fileType.startsWith(".") ? fileType : "." + fileType);
+
+                    PutObjectRequest request = PutObjectRequest.builder()
+                            .bucket(properties.getBucket())
+                            .key(key)
+                            .build();
+
+                    s3Client.putObject(request, RequestBody.fromInputStream(inputStream, inputStream.available()));
+                    return properties.getBaseUrl() + "/" + key;
+                })
+                .subscribeOn(Schedulers.boundedElastic());
+    }
+
+    private String buildFileName(String originalName) {
+        String suffix = "";
+        if (originalName != null && originalName.contains(".")) {
+            suffix = originalName.substring(originalName.lastIndexOf("."));
+        }
+        return UUID.randomUUID().toString().replace("-", "") + suffix.toLowerCase(Locale.ROOT);
+    }
+
+    private String buildFileUrl(String key) {
+        if (properties.getBaseUrl() != null && !properties.getBaseUrl().isEmpty()) {
+            return properties.getBaseUrl() + "/" + key;
+        }
+        return "https://" + properties.getBucket() + "." + properties.getEndpoint().replace("https://", "").replace("http://", "") + "/" + key;
+    }
+}

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/web/ReactiveFileController.java

@@ -10,7 +10,9 @@
 import org.hswebframework.web.authorization.annotation.ResourceAction;
 import org.hswebframework.web.authorization.exception.AccessDenyException;
 import org.hswebframework.web.file.FileUploadProperties;
+import org.hswebframework.web.file.S3StorageProperties;
 import org.hswebframework.web.file.service.FileStorageService;
+import org.hswebframework.web.file.service.S3FileStorageService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.codec.multipart.FilePart;
 import org.springframework.http.codec.multipart.Part;
@@ -21,6 +23,7 @@
 import reactor.core.publisher.Mono;
 
 import java.io.File;
+import java.io.IOException;
 
 @RestController
 @Resource(id = "file", name = "文件上传")
@@ -33,11 +36,13 @@ public class ReactiveFileController {
 
     private final FileStorageService fileStorageService;
 
+
     public ReactiveFileController(FileUploadProperties properties, FileStorageService fileStorageService) {
         this.properties = properties;
         this.fileStorageService = fileStorageService;
     }
 
+
     @PostMapping("/static")
     @SneakyThrows
     @ResourceAction(id = "upload-static", name = "静态文件")
@@ -51,7 +56,11 @@ public Mono<String> uploadStatic(@RequestPart("file")
                         if (properties.denied(filePart.filename(), filePart.headers().getContentType())) {
                            return Mono.error( new AccessDenyException());
                         }
-                        return fileStorageService.saveFile(filePart);
+                        try {
+                            return fileStorageService.saveFile(filePart);
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
                     } else {
                         return Mono.error(() -> new IllegalArgumentException("[file] part is not a file"));
                     }

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/web/S3FileController.java

@@ -0,0 +1,66 @@
+package org.hswebframework.web.file.web;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.enums.ParameterStyle;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.hswebframework.web.authorization.annotation.Resource;
+import org.hswebframework.web.authorization.annotation.ResourceAction;
+import org.hswebframework.web.authorization.exception.AccessDenyException;
+import org.hswebframework.web.file.FileUploadProperties;
+import org.hswebframework.web.file.S3StorageProperties;
+import org.hswebframework.web.file.service.FileStorageService;
+import org.springframework.http.codec.multipart.FilePart;
+import org.springframework.http.codec.multipart.Part;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import reactor.core.publisher.Mono;
+
+import java.io.IOException;
+
+@RestController
+@Resource(id = "ossFile", name = "文件上传")
+@Slf4j
+@RequestMapping("/oss/file")
+@Tag(name = "文件上传")
+public class S3FileController {
+    private final S3StorageProperties properties;
+
+    private final FileStorageService fileStorageService;
+
+
+    public S3FileController(S3StorageProperties properties, FileStorageService fileStorageService) {
+        this.properties = properties;
+        this.fileStorageService = fileStorageService;
+    }
+
+
+    @PostMapping("/static")
+    @SneakyThrows
+    @ResourceAction(id = "upload-static", name = "静态文件")
+    @Operation(summary = "上传静态文件")
+    public Mono<String> uploadStatic(@RequestPart("file")
+                                     @Parameter(name = "file", description = "文件", style = ParameterStyle.FORM) Mono<Part> partMono) {
+        return partMono
+                .flatMap(part -> {
+                    if (part instanceof FilePart) {
+                        FilePart filePart = ((FilePart) part);
+                        if (properties.denied(filePart.filename(), filePart.headers().getContentType())) {
+                            return Mono.error( new AccessDenyException());
+                        }
+                        try {
+                            return fileStorageService.saveFile(filePart);
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
+                    } else {
+                        return Mono.error(() -> new IllegalArgumentException("[file] part is not a file"));
+                    }
+                });
+
+    }
+}

hsweb-system/hsweb-system-file/src/test/java/org/hswebframework/web/file/web/ReactiveFileControllerTest.java

@@ -29,6 +29,7 @@ public class ReactiveFileControllerTest {
 
     static {
         System.setProperty("hsweb.file.upload.static-file-path","./target/upload");
+        System.setProperty("file.storage", "local");
 //        System.setProperty("hsweb.file.upload.use-original-file-name","true");
     }