第2-3-3章 文件处理策略-文件存储服务系统-nginx/fastDFS/minio/阿里云oss/七牛云oss
- 第2-1-2章 传统方式安装FastDFS-附FastDFS常用命令
第2-1-3章 docker-compose安装FastDFS,实现文件存储服务
第2-1-5章 docker安装MinIO实现文件存储服务-springboot整合minio-minio全网最全的资料全套代码及资料全部完整提供,点此处下载
由于我们当前的文件服务需要给客户端提供统一的服务接口,这就需要文件服务屏蔽底层的具体文件存储方式,所以对文件处理策略进行如下类和接口的设计:5.2.1 FileStrategy
FileStrategy是文件处理策略顶层接口,是对文件处理的顶层抽象,具体代码如下:
package com.itheima.pinda.file.strategy; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.entity.File; import org.springframework.web.multipart.MultipartFile; import java.util.List; /** * 文件策略接口 */ public interface FileStrategy { /** * 文件上传 * * @param file 文件 * @return 文件对象 */ File upload(MultipartFile file); /** * 删除文件 * * @param list 列表 */ boolean delete(List
list); } 5.2.2 AbstractFileStrategy
AbstractFileStrategy是抽象文件策略处理类,实现了FileStrategy接口。
AbstractFileStrategy实现主要的文件上传、删除的处理流程,例如异常情况的判断,文件对象的封装等,但是真正上传的处理过程需要其子类来完成,因为不同的存储方案处理方式是不同的。
package com.itheima.pinda.file.strategy.impl; import com.itheima.pinda.exception.BizException; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.entity.File; import com.itheima.pinda.file.enumeration.IconType; import com.itheima.pinda.file.properties.FileServerProperties; import com.itheima.pinda.file.strategy.FileStrategy; import com.itheima.pinda.file.utils.FileDataTypeUtil; import com.itheima.pinda.utils.DateUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.multipart.MultipartFile; import java.time.LocalDateTime; import java.util.List; import static com.itheima.pinda.exception.code.ExceptionCode.BASE_VALID_PARAM; /** * 文件抽象策略处理类 */ @Slf4j public abstract class AbstractFileStrategy implements FileStrategy { private static final String FILE_SPLIT = "."; @Autowired protected FileServerProperties fileProperties; protected FileServerProperties.Properties properties; /** * 上传文件 * * @param multipartFile * @return */ @Override public File upload(MultipartFile multipartFile) { try { if (!multipartFile.getOriginalFilename().contains(FILE_SPLIT)) { throw BizException.wrap(BASE_VALID_PARAM.build("缺少后缀名")); } File file = File.builder() .isDelete(false). submittedFileName(multipartFile.getOriginalFilename()) .contextType(multipartFile.getContentType()) .dataType(FileDataTypeUtil.getDataType(multipartFile.getContentType())) .size(multipartFile.getSize()) .ext(FilenameUtils.getExtension(multipartFile.getOriginalFilename())) .build(); file.setIcon(IconType.getIcon(file.getExt()).getIcon()); setDate(file); uploadFile(file, multipartFile); return file; } catch (Exception e) { log.error("e={}", e); throw BizException.wrap(BASE_VALID_PARAM.build("文件上传失败")); } } /** * 获取下载地址前缀 */ protected String getUriPrefix() { if (StringUtils.isNotEmpty(properties.getUriPrefix())) { return properties.getUriPrefix(); } else { return properties.getEndpoint(); } } /** * 具体类型执行上传操作 * * @param file * @param multipartFile * @throws Exception */ protected abstract void uploadFile(File file, MultipartFile multipartFile) throws Exception; private void setDate(File file) { LocalDateTime now = LocalDateTime.now(); file.setCreateMonth(DateUtils.formatAsYearMonthEn(now)) .setCreateWeek(DateUtils.formatAsYearWeekEn(now)) .setCreateDay(DateUtils.formatAsDateEn(now)); } @Override public boolean delete(List
list) { if (list.isEmpty()) { return true; } boolean flag = false; for (FileDeleteDO file : list) { try { delete(file); flag = true; } catch (Exception e) { log.error("删除文件失败", e); } } return flag; } /** * 具体执行删除方法, 无需处理异常 */ protected abstract void delete(FileDeleteDO file); } 5.2.3 LocalServiceImpl
LocalServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为本地时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义LocalServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage; import cn.hutool.core.util.StrUtil; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.entity.File; import com.itheima.pinda.file.properties.FileServerProperties; import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy; import com.itheima.pinda.utils.StrPool; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.UUID; import static com.itheima.pinda.utils.DateUtils.DEFAULT_MONTH_FORMAT_SLASH; /** * 本地上传配置 */ @EnableConfigurationProperties(FileServerProperties.class) @Configuration @ConditionalOnProperty(name = "pinda.file.type", havingValue = "LOCAL") @Slf4j public class LocalAutoConfigure { /** * 本地文件策略处理类 */ @Service public class LocalServiceImpl extends AbstractFileStrategy { private void buildClient() { properties = fileProperties.getLocal(); } /** * 上传文件 * @param file * @param multipartFile * @throws Exception */ @Override protected void uploadFile(File file, MultipartFile multipartFile) throws Exception { buildClient(); //生成文件名 String fileName = UUID.randomUUID().toString() + StrPool.DOT + file.getExt(); //日期文件夹,例如:2020\04 String relativePath = Paths.get(LocalDate.now().format(DateTimeFormatter.ofPattern(DEFAULT_MONTH_FORMAT_SLASH))).toString(); // web服务器存放的绝对路径,例如:D:\\uploadFiles\\oss-file-service\\2020\\04 String absolutePath = Paths.get(properties.getEndpoint(), properties.getBucketName(), relativePath).toString(); //目标输出文件 java.io.File outFile = new java.io.File(Paths.get(absolutePath, fileName).toString()); //向目标文件写入数据 org.apache.commons.io.FileUtils.writeByteArrayToFile(outFile, multipartFile.getBytes()); String url = new StringBuilder(getUriPrefix()) .append(StrPool.SLASH) .append(properties.getBucketName()) .append(StrPool.SLASH) .append(relativePath) .append(StrPool.SLASH) .append(fileName) .toString(); //替换掉windows环境的\路径 url = StrUtil.replace(url, "\\\\", StrPool.SLASH); url = StrUtil.replace(url, "\\", StrPool.SLASH); file.setUrl(url); file.setFilename(fileName); file.setRelativePath(relativePath); } /** * 文件删除 * @param file */ @Override protected void delete(FileDeleteDO file) { java.io.File ioFile = new java.io.File(Paths.get(properties.getEndpoint(), properties.getBucketName(), file.getRelativePath(), file.getFileName()).toString()); org.apache.commons.io.FileUtils.deleteQuietly(ioFile); } } }
通过上面的代码可以看到,在进行文件上传和文件删除时都会使用到配置文件中的配置项,关于本地文件处理策略的配置如下:
pinda: mysql: database: pd_files nginx: ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口 file: type: LOCAL local: uriPrefix: http://${pinda.nginx.ip}:${pinda.nginx.port} bucket-name: oss-file-service endpoint: D:\soft\nginx-1.23.0\uploadFiles
5.2.4 FastDfsServiceImpl
FastDfsServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为FastDFS时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义FastDfsServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage; import com.github.tobato.fastdfs.domain.fdfs.StorePath; import com.github.tobato.fastdfs.service.FastFileStorageClient; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.entity.File; import com.itheima.pinda.file.properties.FileServerProperties; import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; /** * FastDFS配置 */ @EnableConfigurationProperties(FileServerProperties.class) @Configuration @Slf4j @ConditionalOnProperty(name = "pinda.file.type", havingValue = "FAST_DFS") public class FastDfsAutoConfigure { /** * FastDFS文件策略处理类 */ @Service public class FastDfsServiceImpl extends AbstractFileStrategy { @Autowired private FastFileStorageClient storageClient; //操作FastDFS的客户端 /** * 上传文件 * @param file * @param multipartFile * @throws Exception */ @Override protected void uploadFile(File file, MultipartFile multipartFile) throws Exception { //调用FastDFS客户端将文件上传到FastDFS StorePath storePath = storageClient.uploadFile(multipartFile.getInputStream(), multipartFile.getSize(), file.getExt(), null); file.setUrl(fileProperties.getUriPrefix() + storePath.getFullPath()); file.setGroup(storePath.getGroup()); file.setPath(storePath.getPath()); } /** * 文件删除 * @param file */ @Override protected void delete(FileDeleteDO file) { //调用FastDFS客户端删除文件 storageClient.deleteFile(file.getGroup(), file.getPath()); } } }
通过上面代码可以看到要使用FastDFS提供的客户端FastFileStorageClient来实现文件的上传和删除,这就需要在文件服务对应的配置文件中进行如下配置:
pinda: mysql: database: pd_files nginx: ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口 file: type: FAST_DFS uriPrefix: http://172.17.0.115:8188/ #存储类型为FAST_DFS时使用 #FAST_DFS配置 fdfs: soTimeout: 1500 connectTimeout: 600 thumb-image: width: 150 height: 150 tracker-list: - 172.17.0.115:22122 pool: #从池中借出的对象的最大数目 max-total: 153 max-wait-millis: 102 jmx-name-base: 1 jmx-name-prefix: 1
5.2.5 AliServiceImpl
AliServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为阿里云OSS时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义AliServiceImpl,可以参照阿里云OSS官方提供的示例代码(https://help.aliyun.com/document_detail/32011.html?spm=a2c4g.11186623.6.769.652763282djHGw)进行文件的上传和删除。
具体代码如下:
package com.itheima.pinda.file.storage; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSONObject; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; import com.aliyun.oss.model.*; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.entity.File; import com.itheima.pinda.file.properties.FileServerProperties; import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy; import com.itheima.pinda.utils.StrPool; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.UUID; import static com.itheima.pinda.utils.DateUtils.DEFAULT_MONTH_FORMAT_SLASH; /** * 阿里云OSS配置 */ @EnableConfigurationProperties(FileServerProperties.class) @Configuration @Slf4j @ConditionalOnProperty(name = "pinda.file.type", havingValue = "ALI") public class AliOssAutoConfigure { /** * 阿里云OSS文件策略处理类 */ @Service public class AliServiceImpl extends AbstractFileStrategy { /** * 构建阿里云OSS客户端 * @return */ private OSS buildClient() { properties = fileProperties.getAli(); return new OSSClientBuilder(). build(properties.getEndpoint(), properties.getAccessKeyId(), properties.getAccessKeySecret()); } protected String getUriPrefix() { if (StringUtils.isNotEmpty(properties.getUriPrefix())) { return properties.getUriPrefix(); } else { String prefix = properties. getEndpoint(). contains("https://") ? "https://" : "http://"; return prefix + properties.getBucketName() + "." + properties.getEndpoint().replaceFirst(prefix, ""); } } /** * 上传文件 * @param file * @param multipartFile * @throws Exception */ @Override protected void uploadFile(File file, MultipartFile multipartFile) throws Exception { OSS client = buildClient(); //获得OSS空间名称 String bucketName = properties.getBucketName(); if (!client.doesBucketExist(bucketName)) { //创建存储空间 client.createBucket(bucketName); } //生成文件名 String fileName = UUID.randomUUID().toString() + StrPool.DOT + file.getExt(); //日期文件夹,例如:2020\04 String relativePath = Paths.get(LocalDate.now(). format(DateTimeFormatter. ofPattern(DEFAULT_MONTH_FORMAT_SLASH))). toString(); // web服务器存放的相对路径 String relativeFileName = relativePath + StrPool.SLASH + fileName; relativeFileName = StrUtil.replace(relativeFileName, "\\\\", StrPool.SLASH); relativeFileName = StrUtil.replace(relativeFileName, "\\", StrPool.SLASH); //对象元数据 ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentDisposition("attachment;fileName=" + file.getSubmittedFileName()); metadata.setContentType(file.getContextType()); //上传请求对象 PutObjectRequest request = new PutObjectRequest(bucketName, relativeFileName, multipartFile.getInputStream(), metadata); //上传文件到阿里云OSS空间 PutObjectResult result = client.putObject(request); log.info("result={}", JSONObject.toJSONString(result)); String url = getUriPrefix() + StrPool.SLASH + relativeFileName; url = StrUtil.replace(url, "\\\\", StrPool.SLASH); url = StrUtil.replace(url, "\\", StrPool.SLASH); // 写入文件表 file.setUrl(url); file.setFilename(fileName); file.setRelativePath(relativePath); file.setGroup(result.getETag()); file.setPath(result.getRequestId()); //关闭阿里云OSS客户端 client.shutdown(); } /** * 文件删除 * @param file */ @Override protected void delete(FileDeleteDO file) { OSS client = buildClient(); //获得OSS空间名称 String bucketName = properties.getBucketName(); // 删除文件 client.deleteObject(bucketName, file.getRelativePath() + StrPool.SLASH + file.getFileName()); //关闭阿里云OSS客户端 client.shutdown(); } } }
通过上面代码可以看到要使用阿里云OSS提供的客户端OSS来实现文件的上传和删除,这就需要在文件服务对应的配置文件中进行如下配置:
pinda: mysql: database: pd_files nginx: ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口 file: type: ALI ali: # 请填写自己的阿里云存储配置 bucket-name: bladex-loan endpoint: http://oss-cn-qingdao.aliyuncs.com access-key-id: LTAI4FhtimFAiz6iLGJSiJui access-key-secret: SsU15qaPwpF1x5xMqwc0XzGuY92fnc
5.2.6 MinioServiceImpl
MinioServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为Minio时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义MinioServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.itheima.pinda.base.R; import com.itheima.pinda.file.domain.FileDeleteDO; import com.itheima.pinda.file.dto.BucketPolicyConfigDTO; import com.itheima.pinda.file.dto.chunk.FileChunksMergeDTO; import com.itheima.pinda.file.entity.File; import com.itheima.pinda.file.properties.FileServerProperties; import com.itheima.pinda.file.strategy.impl.AbstractFileChunkStrategy; import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy; import com.itheima.pinda.utils.DateUtils; import com.itheima.pinda.utils.StrPool; import io.minio.BucketExistsArgs; import io.minio.ComposeObjectArgs; import io.minio.ComposeSource; import io.minio.GetObjectArgs; import io.minio.GetPresignedObjectUrlArgs; import io.minio.ListObjectsArgs; import io.minio.MakeBucketArgs; import io.minio.MinioClient; import io.minio.ObjectWriteArgs; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; import io.minio.Result; import io.minio.SetBucketPolicyArgs; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.http.Method; import io.minio.messages.Item; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.Nullable; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.InputStream; import java.io.SequenceInputStream; import java.nio.ByteBuffer; import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.Vector; import java.util.concurrent.TimeUnit; /** * @Author: 郭浩伟 qq:912161367 * @Date: 2022/11/6 0006 19:44 * @Description: minio */ @Configuration @Slf4j @EnableConfigurationProperties(FileServerProperties.class) @ConditionalOnProperty(name = "pinda.file.type", havingValue = "MINIO") public class MinioAutoConfigure { private MinioClient minioClient; private String bucketName; private String endpoint; private static final int DEFAULT_EXPIRY_TIME = 7 * 24 * 3600; /** * 构建minioClient * * @return */ private void buildClient(FileServerProperties fileProperties) { //加载配置文件相关信息 FileServerProperties.Properties properties = fileProperties.getMinio(); endpoint = properties.getEndpoint(); String accessKey = properties.getAccessKey(); String secretKey = properties.getSecretKey(); this.bucketName = properties.getBucketName(); //创建一个MinIO的Java客户端 minioClient = MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) .build(); } /** * 本地文件策略处理类 */ @Service public class MinioServiceImpl extends AbstractFileStrategy { /** * 文件上传抽象方法,需要由当前类的子类来实现 * * @param file * @param multipartFile * @return */ @Override public void uploadFile(File file, MultipartFile multipartFile) throws Exception { MinioAutoConfigure.this.buildClient(fileProperties); //生成文件名 此处并未用原始文件名multipartFile.getOriginalFilename(),因为同名文件会覆盖 String fileName = UUID.randomUUID() + StrPool.DOT + file.getExt(); String objectName = doReName(fileName, file); MinioAutoConfigure.this.putObject(bucketName, multipartFile, objectName); log.info("文件上传成功!"); } /** * 文件删除抽象方法,需要当前类的子类来实现 * * @param fileDeleteDO */ @Override public void delete(FileDeleteDO fileDeleteDO) throws Exception { MinioAutoConfigure.this.buildClient(fileProperties); // 设置存储对象名称 String objectName = Paths.get(fileDeleteDO.getRelativePath(), fileDeleteDO.getFileName()).toString(); // 替换掉windows环境的\路径 objectName = StrUtil.replace(objectName, "\\\\", StrPool.SLASH); objectName = StrUtil.replace(objectName, "\\", StrPool.SLASH); // 执行删除操作 if (bucketExists(bucketName)) { minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build()); } } } }
通过上面的代码可以看到,在进行文件上传和文件删除时都会使用到配置文件中的配置项,关于Minio文件处理策略的配置如下:
pinda: mysql: database: pd_files nginx: ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口 file: type: MINIO minio: endpoint: http://192.168.86.101:9000 #MinIO服务所在地址 bucketName: file #存储桶名称 accessKey: admin #访问的key secretKey: admin123456 #访问的秘钥
全套代码及资料全部完整提供,点此处下载