diff --git a/jianshui-admin/src/main/java/com/jianshui/api/apiutils/DataViewManYiController.java b/jianshui-admin/src/main/java/com/jianshui/api/apiutils/DataViewManYiController.java new file mode 100644 index 0000000..ee918b9 --- /dev/null +++ b/jianshui-admin/src/main/java/com/jianshui/api/apiutils/DataViewManYiController.java @@ -0,0 +1,148 @@ +package com.jianshui.api.apiutils; + +import cn.hutool.core.collection.ListUtil; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import cn.hutool.poi.excel.ExcelReader; +import cn.hutool.poi.excel.ExcelUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.jianshui.common.core.domain.AjaxResult; +import com.jianshui.invoice.domain.Invoice; +import com.jianshui.invoice.mapper.InvoiceMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author kk + * 满意数据清洗 + * @date 2023年11月24日 14:24 + */ +@RestController +@Slf4j +public class DataViewManYiController { + + @Resource + private InvoiceMapper invoiceMapper; + + + + @GetMapping("/getDataManYi") + public AjaxResult getDataManYi(MultipartFile file) throws IOException { + ExcelReader reader = ExcelUtil.getReader(file.getInputStream()); + List invoiceList = reader.readAll(Invoice.class); + log.info("总条数:{}", invoiceList.size()); + log.info("下载开始={}", new Date()); + + // 计算线程数 + int threadCount = calculateThreadCount(); + log.info("线程数:{}", threadCount); + + // 拆分任务列表 + List> invoiceListGroup = ListUtil.splitAvg(invoiceList, threadCount); + + // 创建线程池 + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + // 计数器 + AtomicInteger downloadCount = new AtomicInteger(0); + // 提交任务 + AtomicInteger taskCount = new AtomicInteger(0); + for (List invoices : invoiceListGroup) { + executor.execute(() -> { + for (Invoice invoice : invoices) { + try { + downloadInvoice(invoice); + downloadCount.incrementAndGet(); + } catch (IOException e) { + log.error("下载发票失败,记录id,invoiceid={},fphm={}", invoice.getId(), invoice.getFphm(), e); + } + taskCount.incrementAndGet(); + } + }); + } + + // 等待任务完成 + while (taskCount.get() < invoiceList.size()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + log.error("任务等待被中断", e); + break; + } + } + + // 关闭线程池 + executor.shutdown(); + + log.info("总共下载了{}个文件", downloadCount.get()); + return AjaxResult.success("文件下载完成"); + } + + private void downloadInvoice(Invoice invoice) throws IOException { + Date kprq = invoice.getKprq(); + String fphm = invoice.getFphm(); + String fpdm = invoice.getFpdm(); + String fileUrl = invoice.getcOfdUrl(); + + // 检查是否缺少必要信息 + if (ObjectUtil.isNull(kprq) || ObjectUtil.isNull(fphm) || ObjectUtil.isNull(fpdm) || ObjectUtil.isNull(fileUrl)) { + log.warn("发票信息不完整,记录id,invoiceid={},fphm={}", invoice.getId(), invoice.getFphm()); + return; + } + + // 构造文件路径 + String kprqStr = DateUtil.format(kprq, "yyyy-MM-dd"); + String[] kprqArray = kprqStr.split("-"); + String savePath = "C://downloadMY/" + kprqArray[0] + "/" + kprqArray[1] + "/" + kprqArray[2] + "/" + fphm.substring(fphm.length() - 3, fphm.length()) + "/" + fpdm + "_" + fphm + ".ofd"; + File fileRes = new File(savePath); + + // 检查文件是否已存在 + if (fileRes.exists()) { + log.debug("文件已存在,跳过下载:{}", fileRes); + return; + } + + // 下载文件 + HttpResponse response = HttpRequest.get(fileUrl).execute(); + if (response.isOk()) { + InputStream inputStream = response.bodyStream(); + FileUtil.mkdir(fileRes.getParent()); + IoUtil.copy(inputStream, FileUtil.getOutputStream(fileRes)); + log.debug("下载成功:{}", fileRes); + } else { + log.warn("下载失败:{},原因:{}", fileRes, response.getStatus()); + } + } + + private int calculateThreadCount() { + int cpuCores = Runtime.getRuntime().availableProcessors(); + long maxMemory = Runtime.getRuntime().maxMemory(); + double memoryRatio = 0.8; + double diskIORatio = 0.7; + long availableMemory = (long) (maxMemory * memoryRatio); + int availableThreads = (int) (cpuCores * diskIORatio); + int maxThreads = (int) (availableMemory / (1024 * 1024 * 10)); // 每个线程分配10MB内存 + int maxThreadsPerDiskIO = 20; // 每个磁盘IO操作最多分配20个线程 + return Math.min(maxThreads, maxThreadsPerDiskIO); + } + + + + +} diff --git a/jianshui-framework/src/main/java/com/jianshui/framework/config/SecurityConfig.java b/jianshui-framework/src/main/java/com/jianshui/framework/config/SecurityConfig.java index 587208f..22e147a 100644 --- a/jianshui-framework/src/main/java/com/jianshui/framework/config/SecurityConfig.java +++ b/jianshui-framework/src/main/java/com/jianshui/framework/config/SecurityConfig.java @@ -137,6 +137,8 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { // .antMatchers("/*/api-docs").anonymous() // .antMatchers("/webjars/**").anonymous() // .antMatchers("/druid/**").anonymous() + /** 满意数据清洗*/ + .antMatchers("/getDataManYi").anonymous() // 除上面外的所有请求全部需要鉴权认证 .anyRequest().authenticated()