1 package fr.ifremer.quadrige3.core.dao.technical;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 import org.apache.commons.io.FileUtils;
25 import org.apache.commons.io.IOUtils;
26 import org.apache.commons.lang3.time.DurationFormatUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.junit.Assert;
30 import org.junit.Test;
31 import org.springframework.util.DigestUtils;
32 import org.springframework.util.StreamUtils;
33
34 import java.io.*;
35 import java.nio.file.Path;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Random;
40 import java.util.stream.Collectors;
41
42 import static java.nio.file.StandardOpenOption.READ;
43
44
45
46
47 public class FilesTest {
48
49 private static final Log LOG = LogFactory.getLog(FilesTest.class);
50
51 @Test
52 public void splitSmallFile() throws IOException {
53
54
55 Path testFile1 = createTestBinaryFile(1024 * 1024, null);
56
57 List<Path> files = Files.splitFile(testFile1, 1024 * 1024);
58 Assert.assertNotNull(files);
59 Assert.assertEquals(1, files.size());
60
61 Assert.assertEquals(testFile1, files.get(0));
62
63 }
64
65 @Test
66 public void splitAndMergeFile() throws IOException {
67
68
69 Path testFile1 = createTestBinaryFile(1024 * 1024 * 8 + 523189, null);
70 byte[] md5File1 = md5(testFile1);
71
72
73 List<Path> files = Files.splitFile(testFile1, 1024 * 1024);
74 Assert.assertNotNull(files);
75 Assert.assertEquals(9, files.size());
76 Assert.assertEquals(523189, java.nio.file.Files.size(files.get(8)));
77
78 Path testFile2 = createTestBinaryFile(0, null);
79 java.nio.file.Files.delete(testFile2);
80 Assert.assertFalse(java.nio.file.Files.exists(testFile2));
81
82 Files.mergeFiles(files, testFile2);
83
84 byte[] md5File2 = md5(testFile2);
85 Assert.assertArrayEquals(md5File1, md5File2);
86
87
88 List<Path> foundFiles = Files.listOfFilesToMerge(testFile1);
89 Assert.assertNotNull(foundFiles);
90 Assert.assertEquals(9, foundFiles.size());
91 Assert.assertArrayEquals(files.stream().map(Path::getFileName).toArray(), foundFiles.stream().map(Path::getFileName).toArray());
92
93
94 java.nio.file.Files.delete(testFile1);
95 java.nio.file.Files.delete(testFile2);
96 for (Path file : files) {
97 java.nio.file.Files.delete(file);
98 }
99 }
100
101 @Test
102 public void copyTest() throws IOException {
103
104 {
105 long size = 1024 * 1024 * 1000;
106
107 Path source = createTestBinaryFile(size, null);
108 Path destination = source.resolveSibling(source.getFileName().toString() + ".copy");
109 long start = System.currentTimeMillis();
110 FileUtils.copyFile(source.toFile(), destination.toFile());
111 LOG.info(String.format("Copy large file with org.apache.commons.io.FileUtils.copyFile() in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
112 java.nio.file.Files.delete(source);
113 java.nio.file.Files.delete(destination);
114
115 source = createTestBinaryFile(size, null);
116 destination = source.resolveSibling(source.getFileName().toString() + ".copy");
117 start = System.currentTimeMillis();
118 try (InputStream is = new BufferedInputStream(new FileInputStream(source.toFile()));
119 OutputStream os = new BufferedOutputStream(new FileOutputStream(destination.toFile()))) {
120 StreamUtils.copy(is, os);
121 }
122 LOG.info(String.format("Copy large file with org.springframework.util.StreamUtils.copy() and Buffered*Stream in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
123 java.nio.file.Files.delete(source);
124 java.nio.file.Files.delete(destination);
125
126 source = createTestBinaryFile(size, null);
127 destination = source.resolveSibling(source.getFileName().toString() + ".copy");
128 start = System.currentTimeMillis();
129 try (InputStream is = new BufferedInputStream(new FileInputStream(source.toFile()));
130 OutputStream os = new BufferedOutputStream(new FileOutputStream(destination.toFile()))) {
131 IOUtils.copy(is, os);
132 }
133 LOG.info(String.format("Copy large file with org.apache.commons.io.IOUtils.copy() and Buffered*Stream in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
134 java.nio.file.Files.delete(source);
135 java.nio.file.Files.delete(destination);
136
137 source = createTestBinaryFile(size, null);
138 destination = source.resolveSibling(source.getFileName().toString() + ".copy");
139 start = System.currentTimeMillis();
140 try (InputStream is = java.nio.file.Files.newInputStream(source);
141 OutputStream os = java.nio.file.Files.newOutputStream(destination)) {
142 IOUtils.copy(is, os);
143 }
144 LOG.info(String.format("Copy large file with org.apache.commons.io.IOUtils.copy() and NIO streams in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
145 java.nio.file.Files.delete(source);
146 java.nio.file.Files.delete(destination);
147
148 source = createTestBinaryFile(size, null);
149 destination = source.resolveSibling(source.getFileName().toString() + ".copy");
150 start = System.currentTimeMillis();
151 java.nio.file.Files.copy(source, destination);
152 LOG.info(String.format("Copy large file with java.nio.file.Files.copy() in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
153 java.nio.file.Files.delete(source);
154 java.nio.file.Files.delete(destination);
155
156 source = createTestBinaryFile(size, null);
157 destination = source.resolveSibling(source.getFileName().toString() + ".copy");
158 start = System.currentTimeMillis();
159 Files.copyFile(source, destination);
160 LOG.info(String.format("Copy large file with fr.ifremer.quadrige3.core.dao.technical.Files.copyFile in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
161 java.nio.file.Files.delete(source);
162 java.nio.file.Files.delete(destination);
163
164 }
165
166
167 {
168 long[] sizes = new long[]{
169 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
170 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
171 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
172 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
173 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
174 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
175 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
176 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
177 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
178 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
179 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500,
180 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500, 1024 * 100, 1024 * 200, 1024 * 300, 1024 * 400, 1024 * 500
181 };
182
183 List<Path> sources = createMultipleTestBinaryFile(sizes);
184 Map<Path, Path> destinationsBySource = sources.stream().collect(Collectors.toMap(source -> source, source -> source.resolveSibling(source.getFileName().toString() + ".copy")));
185 long start = System.currentTimeMillis();
186 for (Path source : destinationsBySource.keySet()) {
187 FileUtils.copyFile(source.toFile(), destinationsBySource.get(source).toFile());
188 }
189 LOG.info(String.format("Copy small files with org.apache.commons.io.FileUtils.copyFile() in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
190 for (Path source: destinationsBySource.keySet()) {
191 java.nio.file.Files.delete(source);
192 java.nio.file.Files.delete(destinationsBySource.get(source));
193 }
194
195 sources = createMultipleTestBinaryFile(sizes);
196 destinationsBySource = sources.stream().collect(Collectors.toMap(source -> source, source -> source.resolveSibling(source.getFileName().toString() + ".copy")));
197 start = System.currentTimeMillis();
198 for (Path source : destinationsBySource.keySet()) {
199 try (InputStream is = new BufferedInputStream(new FileInputStream(source.toFile()));
200 OutputStream os = new BufferedOutputStream(new FileOutputStream(destinationsBySource.get(source).toFile()))) {
201 IOUtils.copy(is, os);
202 }
203 }
204 LOG.info(String.format("Copy small files with org.apache.commons.io.IOUtils.copy() and buffered streams in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
205 for (Path source: destinationsBySource.keySet()) {
206 java.nio.file.Files.delete(source);
207 java.nio.file.Files.delete(destinationsBySource.get(source));
208 }
209
210 sources = createMultipleTestBinaryFile(sizes);
211 destinationsBySource = sources.stream().collect(Collectors.toMap(source -> source, source -> source.resolveSibling(source.getFileName().toString() + ".copy")));
212 start = System.currentTimeMillis();
213 for (Path source : destinationsBySource.keySet()) {
214 try (InputStream is = java.nio.file.Files.newInputStream(source);
215 OutputStream os = java.nio.file.Files.newOutputStream(destinationsBySource.get(source))) {
216 IOUtils.copy(is, os);
217 }
218 }
219 LOG.info(String.format("Copy small files with org.apache.commons.io.IOUtils.copy() and NIO streams in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
220 for (Path source: destinationsBySource.keySet()) {
221 java.nio.file.Files.delete(source);
222 java.nio.file.Files.delete(destinationsBySource.get(source));
223 }
224
225 sources = createMultipleTestBinaryFile(sizes);
226 destinationsBySource = sources.stream().collect(Collectors.toMap(source -> source, source -> source.resolveSibling(source.getFileName().toString() + ".copy")));
227 start = System.currentTimeMillis();
228 for (Path source : destinationsBySource.keySet()) {
229 java.nio.file.Files.copy(source, destinationsBySource.get(source));
230 }
231 LOG.info(String.format("Copy small files with java.nio.file.Files.copy() in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
232 for (Path source: destinationsBySource.keySet()) {
233 java.nio.file.Files.delete(source);
234 java.nio.file.Files.delete(destinationsBySource.get(source));
235 }
236
237 sources = createMultipleTestBinaryFile(sizes);
238 destinationsBySource = sources.stream().collect(Collectors.toMap(source -> source, source -> source.resolveSibling(source.getFileName().toString() + ".copy")));
239 start = System.currentTimeMillis();
240 for (Path source : destinationsBySource.keySet()) {
241 Files.copyFile(source, destinationsBySource.get(source));
242 }
243 LOG.info(String.format("Copy small files with fr.ifremer.quadrige3.core.dao.technical.Files.copyFile in %s", DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
244 for (Path source: destinationsBySource.keySet()) {
245 java.nio.file.Files.delete(source);
246 java.nio.file.Files.delete(destinationsBySource.get(source));
247 }
248
249 }
250
251 }
252
253 @Test
254 public void zipTest() throws IOException {
255
256
257 Path binFile1 = createTestBinaryFile(1024 * 1024 * 100, ".zip");
258 Path txtFile1 = createTestTextFile(1024 * 1024 * 100);
259 Path binPath1 = binFile1.getParent().resolve("toZip").resolve(binFile1.getFileName());
260 Path txtPath1 = binFile1.getParent().resolve("toZip").resolve(txtFile1.getFileName());
261
262 java.nio.file.Files.createDirectories(binPath1.getParent());
263 java.nio.file.Files.move(binFile1, binPath1);
264 java.nio.file.Files.move(txtFile1, txtPath1);
265 Path zipPath1 = java.nio.file.Files.createTempFile("ZIP", ".zip");
266
267 long start = System.currentTimeMillis();
268 ZipUtils.compressFilesInPath(binPath1.getParent(), zipPath1, false);
269 long rawSize = java.nio.file.Files.size(binPath1) + java.nio.file.Files.size(txtPath1);
270 long zipSize = java.nio.file.Files.size(zipPath1);
271
272 LOG.info(String.format("compression of file %s (size=%s) to file %s (size=%s) : ratio %s%% in %s",
273 binPath1.getParent(), Files.byteCountToDisplaySize(rawSize),
274 zipPath1, Files.byteCountToDisplaySize(zipSize),
275 zipSize * 100 / rawSize,
276 DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - start)));
277
278 java.nio.file.Files.delete(binPath1);
279 java.nio.file.Files.delete(txtPath1);
280 java.nio.file.Files.delete(zipPath1);
281 }
282
283 @Test
284 public void testByteCountToDisplaySizeLong() {
285 Assert.assertEquals(Files.byteCountToDisplaySize(Character.MAX_VALUE), "64 KB");
286 Assert.assertEquals(Files.byteCountToDisplaySize(Short.MAX_VALUE), "32 KB");
287 Assert.assertEquals(Files.byteCountToDisplaySize(Integer.MAX_VALUE), "2 GB");
288 Assert.assertEquals(Files.byteCountToDisplaySize(0), "0 bytes");
289 Assert.assertEquals(Files.byteCountToDisplaySize(1), "1 bytes");
290 Assert.assertEquals(Files.byteCountToDisplaySize(1023), "1023 bytes");
291 Assert.assertEquals(Files.byteCountToDisplaySize(1024), "1 KB");
292 Assert.assertEquals(Files.byteCountToDisplaySize(1030), "1.01 KB");
293 Assert.assertEquals(Files.byteCountToDisplaySize(1224), "1.20 KB");
294 Assert.assertEquals(Files.byteCountToDisplaySize(10240), "10 KB");
295 Assert.assertEquals(Files.byteCountToDisplaySize(24832184), "23.7 MB");
296 Assert.assertEquals(Files.byteCountToDisplaySize(new Long("38174914740")), "35.6 GB");
297 Assert.assertEquals(Files.byteCountToDisplaySize(new Long("1374389534720")), "1.25 TB");
298 }
299
300 private static Path createTestBinaryFile(long ofSize, String extension) throws IOException {
301 int bufferSize = 1024 * 32;
302 Random random = new Random();
303 Path tempFile = java.nio.file.Files.createTempFile("BIN", extension != null ? extension : ".bin");
304
305 if (ofSize > 0) try (OutputStream stream = java.nio.file.Files.newOutputStream(tempFile)) {
306 long remaining = ofSize;
307 while (remaining > 0) {
308 if (remaining < bufferSize) {
309 bufferSize = (int) remaining;
310 }
311 byte[] buffer = new byte[bufferSize];
312 random.nextBytes(buffer);
313 stream.write(buffer);
314 remaining -= bufferSize;
315 }
316 stream.flush();
317 }
318 return tempFile;
319 }
320
321 private List<Path> createMultipleTestBinaryFile(long[] ofSizes) throws IOException {
322 List<Path> files = new ArrayList<>();
323 for (long size : ofSizes) {
324 files.add(createTestBinaryFile(size, null));
325 }
326 return files;
327 }
328
329 private static Path createTestTextFile(long ofSize) throws IOException {
330 int bufferSize = 1024;
331 Random random = new Random();
332 Path tempFile = java.nio.file.Files.createTempFile("TXT", ".txt");
333
334 if (ofSize > 0) try (BufferedWriter stream = java.nio.file.Files.newBufferedWriter(tempFile)) {
335 long remaining = ofSize;
336 while (remaining > 0) {
337 if (remaining < bufferSize) {
338 bufferSize = (int) remaining;
339 }
340 byte[] buffer = new byte[bufferSize];
341 random.nextBytes(buffer);
342 stream.write(new String(buffer) + System.lineSeparator());
343 remaining -= bufferSize * 1.5;
344 }
345 stream.flush();
346 }
347
348 return tempFile;
349 }
350
351 private static byte[] md5(Path file) throws IOException {
352 return DigestUtils.md5Digest(java.nio.file.Files.newInputStream(file, READ));
353 }
354 }