直接show code，话不多说
zip 压缩
1、pythonimport osimport zipfileimport concurrent.futuresdef zip_folder(folder_path): # 获取文件夹名称 folder_name = os.path.basename(folder_path) # 创建压缩文件名 zip_filename = folder_name + '.zip' # 创建压缩文件 with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: # 遍历文件夹下的所有文件和子文件夹 for root, dirs, files in os.walk(folder_path): # 压缩当前文件夹内所有的文件 for file in files: file_path = os.path.join(root, file) print(file_path) zipf.write(file_path, os.path.relpath(file_path, folder_path))def main(): # 指定文件夹路径 target_folder = 'c:/py-test' # 获取文件夹下的所有子文件夹路径 subfolders = [os.path.join(target_folder, name) for name in os.listdir(target_folder) if os.path.isdir(os.path.join(target_folder, name))] # 使用多线程压缩文件夹 with concurrent.futures.ThreadPoolExecutor() as executor: executor.map(zip_folder, subfolders)if __name__ == '__main__': main()
python 多线程：13.4 seconds
2、java ExecutorService
java 多线程之 ExecutorService
import cn.hutool.core.date.StopWatch;import cn.hutool.core.io.FileUtil;import cn.hutool.core.util.ZipUtil;import lombok.extern.slf4j.Slf4j;import java.io.File;import java.util.ArrayList;import java.util.List;import java.util.Objects;import java.util.concurrent.Callable;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;@Slf4jpublic FileDealTask implements Callable<Integer> { private String filePath; private String suffix; public FileDealTask(String filePath,String suffix){ this.filePath = filePath; this.suffix = suffix; } @Override public Integer call() throws Exception { StopWatch stopWatch = new StopWatch(); try { stopWatch.start(); String zipPath = filePath + suffix; ZipUtil.zip(filePath,zipPath); stopWatch.stop(); log.info("{} zip file cost time : {}",filePath,stopWatch.getTotalTimeSeconds()); } catch (Exception e) { stopWatch.stop(); log.info("{} zip file cost time : {}",filePath,stopWatch.getTotalTimeSeconds()); return 0; } return 1; } public static void main(String[] args) throws Exception { File rootPath = new File("c:/test"); File[] rootSecondDirs = rootPath.listFiles(); List<File> unzipFileDirs = new ArrayList<>(); for (File targetZipDir : Objects.requireNonNull(rootSecondDirs)) { if(targetZipDir.getAbsolutePath().contains(".zip")){ FileUtil.del(targetZipDir); } else { unzipFileDirs.add(targetZipDir); } } StopWatch stopWatch = new StopWatch(); stopWatch.start(); ExecutorService executorService = Executors.newFixedThreadPool(10); List<FileDealTask> tasks = new ArrayList<>(); for (File targetZipDir : Objects.requireNonNull(unzipFileDirs)) { tasks.add(new FileDealTask(targetZipDir.getAbsolutePath(),".zip")); } List<Future<Integer>> results = executorService.invokeAll(tasks); Integer total = 0; for (Future<Integer> result : results) { total += result.get(); } log.info("预处理的文件个数: {},已处理的文件个数: {}",unzipFileDirs.size(),total); executorService.shutdown(); stopWatch.stop(); log.info("zip file total time : {} seconds",stopWatch.getTotalTimeSeconds()); log.info("------------------"); handleSingleThreadZipFile(unzipFileDirs,"-single.zip"); } static void handleSingleThreadZipFile(List<File> unzipFileDirs,String suffix){ StopWatch mainStopWatch = new StopWatch(); mainStopWatch.start(); unzipFileDirs.forEach(zipFileDir -> { StopWatch stopWatch = new StopWatch(); String filePath = zipFileDir.getAbsolutePath(); try { stopWatch.start(); String zipPath = filePath + suffix; ZipUtil.zip(filePath,zipPath); stopWatch.stop(); log.info("[single] {} zip file cost time : {}",filePath,stopWatch.getTotalTimeSeconds()); } catch (Exception e) { stopWatch.stop(); log.info("[single] {} zip file cost time : {}",filePath,stopWatch.getTotalTimeSeconds()); } }); mainStopWatch.stop(); log.info("[single] total zip file total time : {} seconds",mainStopWatch.getTotalTimeSeconds()); }}15:52:27.077 [pool-1-thread-9] INFO FileDealTask - c:\test\mqttServices zip file cost time : 0.102432515:52:32.028 [pool-1-thread-1] INFO FileDealTask - c:\test\business-general zip file cost time : 5.066937915:52:32.803 [pool-1-thread-5] INFO FileDealTask - c:\test\foundation-data-analyse zip file cost time : 5.841317115:52:32.843 [pool-1-thread-2] INFO FileDealTask - c:\test\data-calculate zip file cost time : 5.8822315:52:32.880 [pool-1-thread-10] INFO FileDealTask - c:\test\rain zip file cost time : 5.918683215:52:32.905 [pool-1-thread-9] INFO FileDealTask - c:\test\rain-calculate zip file cost time : 5.817763415:52:33.891 [pool-1-thread-4] INFO FileDealTask - c:\test\forecast zip file cost time : 6.929748415:52:34.191 [pool-1-thread-6] INFO FileDealTask - c:\test\foundation-map-config zip file cost time : 7.229707115:52:34.872 [pool-1-thread-8] INFO FileDealTask - c:\test\monitor zip file cost time : 7.911350415:52:36.186 [pool-1-thread-1] INFO FileDealTask - c:\test\ScadaApiServer zip file cost time : 4.158894915:52:37.313 [pool-1-thread-10] INFO FileDealTask - c:\test\security-auth zip file cost time : 4.432522915:52:37.496 [pool-1-thread-2] INFO FileDealTask - c:\test\security-admin zip file cost time : 4.653267415:52:37.810 [pool-1-thread-9] INFO FileDealTask - c:\test\security-gateway zip file cost time : 4.904718815:52:38.481 [pool-1-thread-3] INFO FileDealTask - c:\test\event zip file cost time : 11.519985815:52:39.351 [pool-1-thread-7] INFO FileDealTask - c:\test\hs-message zip file cost time : 12.390195715:52:42.060 [pool-1-thread-5] INFO FileDealTask - c:\test\scenario zip file cost time : 9.255846415:52:42.060 [main] INFO FileDealTask - 预处理的文件个数: 16,已处理的文件个数: 1615:52:42.061 [main] INFO FileDealTask - zip file total time : 15.1030676 seconds15:52:42.061 [main] INFO FileDealTask - ------------------15:52:44.274 [main] INFO FileDealTask - [single] c:\test\business-general zip file cost time : 2.209135815:52:46.867 [main] INFO FileDealTask - [single] c:\test\data-calculate zip file cost time : 2.59288415:52:52.871 [main] INFO FileDealTask - [single] c:\test\event zip file cost time : 6.003600915:52:55.884 [main] INFO FileDealTask - [single] c:\test\forecast zip file cost time : 3.012893115:52:58.408 [main] INFO FileDealTask - [single] c:\test\foundation-data-analyse zip file cost time : 2.523969815:53:01.566 [main] INFO FileDealTask - [single] c:\test\foundation-map-config zip file cost time : 3.157484815:53:08.375 [main] INFO FileDealTask - [single] c:\test\hs-message zip file cost time : 6.8095315:53:12.125 [main] INFO FileDealTask - [single] c:\test\monitor zip file cost time : 3.749917615:53:12.128 [main] INFO FileDealTask - [single] c:\test\mqttServices zip file cost time : 0.003008915:53:14.811 [main] INFO FileDealTask - [single] c:\test\rain zip file cost time : 2.682898515:53:17.853 [main] INFO FileDealTask - [single] c:\test\rain-calculate zip file cost time : 3.042070115:53:20.651 [main] INFO FileDealTask - [single] c:\test\ScadaApiServer zip file cost time : 2.796622315:53:30.241 [main] INFO FileDealTask - [single] c:\test\scenario zip file cost time : 9.589876815:53:33.647 [main] INFO FileDealTask - [single] c:\test\security-admin zip file cost time : 3.40675815:53:37.601 [main] INFO FileDealTask - [single] c:\test\security-auth zip file cost time : 3.952847815:53:42.291 [main] INFO FileDealTask - [single] c:\test\security-gateway zip file cost time : 4.690283715:53:42.291 [main] INFO FileDealTask - [single] total zip file total time : 60.2297293 seconds
单线程 cost-time = 多线程 cost-time * 4
效率提升四倍 非常明显
