View Javadoc
1   package net.sumaris.core.util;
2   
3   /*-
4    * #%L
5    * SUMARiS:: Core shared
6    * %%
7    * Copyright (C) 2018 - 2019 SUMARiS Consortium
8    * %%
9    * This program is free software: you can redistribute it and/or modify
10   * it under the terms of the GNU General Public License as
11   * published by the Free Software Foundation, either version 3 of the
12   * License, or (at your option) any later version.
13   * 
14   * This program is distributed in the hope that it will be useful,
15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17   * GNU General Public License for more details.
18   * 
19   * You should have received a copy of the GNU General Public
20   * License along with this program.  If not, see
21   * <http://www.gnu.org/licenses/gpl-3.0.html>.
22   * #L%
23   */
24  
25  import com.google.common.base.Preconditions;
26  import com.google.common.collect.Lists;
27  import net.sumaris.core.exception.SumarisTechnicalException;
28  import org.apache.commons.lang3.StringUtils;
29  import org.slf4j.Logger;
30  import org.slf4j.LoggerFactory;
31  import org.springframework.core.io.Resource;
32  import org.springframework.core.io.ResourceLoader;
33  
34  import java.io.*;
35  import java.nio.file.*;
36  import java.nio.file.attribute.BasicFileAttributes;
37  import java.text.DecimalFormat;
38  import java.text.DecimalFormatSymbols;
39  import java.text.NumberFormat;
40  import java.text.ParseException;
41  import java.util.Collection;
42  import java.util.List;
43  import java.util.Map;
44  import java.util.function.Predicate;
45  import java.util.regex.Matcher;
46  import java.util.regex.Pattern;
47  
48  public class Files {
49  
50  	private static final Logger log = LoggerFactory.getLogger(Files.class);
51  
52  	public static final String TEMPORARY_FILE_DEFAULT_EXTENSION =".tmp";
53  
54  	protected static NumberFormat numberFormat = NumberFormat.getInstance();
55  	protected static char decimalSeparator = '\0';
56  	protected static char inverseDecimalSeparator = ',';
57  	protected static Pattern scientificExpressionPattern = Pattern.compile("([0-9.]+)([Ee][-+])([0-9]+)");
58  	protected static DecimalFormat decimalFormat = (DecimalFormat) DecimalFormat.getInstance();
59  
60  	static {
61  		DecimalFormatSymbols decimalFormatSymbols = new DecimalFormatSymbols();
62  		decimalSeparator = decimalFormatSymbols.getDecimalSeparator();
63  		if (decimalSeparator == ',') {
64  			inverseDecimalSeparator = '.';
65  		}
66  		decimalFormat.applyPattern("#0.#########");
67  	}
68  
69  	public static void checkExists(File file) throws FileNotFoundException{
70  		Preconditions.checkNotNull(file);
71  		if (file.exists() == false) {
72  			throw new FileNotFoundException("File not exists: " + file.getAbsolutePath());
73  		}
74  	}
75  
76  	public static void convert(File sourceFile, String sourceCharset, File destFile, String destCharset) throws IOException {
77  
78  		log.debug(String.format("Converting file to encoding %s: %s", destCharset, sourceFile.getPath()));
79  		BufferedInputStream bis = new BufferedInputStream(new FileInputStream(sourceFile));
80  		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destFile));
81  
82  		byte[] buf = new byte[2048];
83  		int len = 0;
84  		while ((len = bis.read(buf)) != -1) {
85  			String ustf8String = new String(buf, 0, len, sourceCharset);
86  
87  			bos.write(ustf8String.getBytes(destCharset));
88  		}
89  
90  		bos.close();
91  		bis.close();
92  	}
93  
94  	public static void replaceAll(File sourceFile, String regex, File destFile, String replacement) throws IOException {
95  
96  		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
97  		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
98  
99  		String line = null;
100 		while ((line = reader.readLine()) != null) {
101 			writer.write(line.replaceAll(regex, replacement));
102 			writer.newLine();
103 		}
104 
105 		reader.close();
106 		writer.close();
107 	}
108 
109 	public static void replaceAllInHeader(File sourceFile, File destFile, Map<String, String> regexReplacementMap) throws IOException {
110 		replaceAll(sourceFile, destFile, regexReplacementMap, 1, 1);
111 	}
112 
113 	public static void replaceAll(File sourceFile, Map<String, String> regexReplacementMap, int startRowNumber, int endRowNumber) throws IOException {
114 		// Generate a temp file name
115 		File destFile = getNewTemporaryFile(sourceFile);
116 
117 		// Do the replace all
118 		replaceAll(sourceFile, destFile, regexReplacementMap, startRowNumber, endRowNumber);
119 
120 		// Override the source file content
121 		sourceFile.delete();
122 		destFile.renameTo(sourceFile);
123 	}
124 
125 	public static void replaceAll(File sourceFile, File destFile, Map<String, String> regexReplacementMap, int startRowNumber, int endRowNumber) throws IOException {
126 		Preconditions.checkArgument(startRowNumber >= 0);
127 		Preconditions.checkArgument(endRowNumber == -1 || endRowNumber >= 1);
128 
129 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
130 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
131 
132 		String line;
133 		int counter = 0;
134 		while ((line = reader.readLine()) != null) {
135 			counter++;
136 			boolean apply = (counter >= startRowNumber) && (endRowNumber == -1 || counter <= endRowNumber);
137 
138 			// Replace line content
139 			if (apply) {
140 				line = regexReplacementMap.keySet().stream().reduce(line, (res, regexp) -> {
141 						try {
142 							return res.replaceAll(regexp, regexReplacementMap.get(regexp));
143 						} catch(Throwable t) {
144 							throw new SumarisTechnicalException(
145 									String.format("Error on replacement {%s}->{%s}: %s\n", regexp, regexReplacementMap.get(regexp), t.getMessage()),
146 									t);
147 						}
148 				});
149 			}
150 			writer.write(line);
151 			writer.newLine();
152 		}
153 
154 		reader.close();
155 		writer.close();
156 	}
157 
158 	public static void filter(File sourceFile, File destFile, Predicate<String> filter) throws IOException {
159 
160 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
161 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
162 
163 		String line;
164 		while ((line = reader.readLine()) != null) {
165 			if (filter.test(line)) {
166 				writer.write(line);
167 				writer.newLine();
168 			}
169 		}
170 
171 		reader.close();
172 		writer.close();
173 	}
174 
175 	public static void replaceAll(File sourceFile, String regex, String replacement) throws IOException {
176 
177 		// Generate a temp file name
178 		File destFile = getNewTemporaryFile(sourceFile);
179 
180 		// Do the replace all
181 		replaceAll(sourceFile, regex, destFile, replacement);
182 
183 		// Override the source file content
184 		sourceFile.delete();
185 		destFile.renameTo(sourceFile);
186 	}
187 
188 	public static void replaceAllExcelScientificExpression(File sourceFile, File destFile) throws IOException, ParseException {
189 
190 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
191 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
192 
193 		String line = null;
194 		while ((line = reader.readLine()) != null) {
195 			line = replaceAllExcelScientificExpression(line);
196 			writer.write(line);
197 			writer.newLine();
198 		}
199 
200 		reader.close();
201 		writer.close();
202 	}
203 
204 	public static void replaceAllExcelScientificExpression(File sourceFile) throws IOException, ParseException {
205 		File destFile = getNewTemporaryFile(sourceFile);
206 
207 		// Do the replacement
208 		replaceAllExcelScientificExpression(sourceFile, destFile);
209 
210 		// Override the source file content
211 		sourceFile.delete();
212 		destFile.renameTo(sourceFile);
213 	}
214 
215 	private static String replaceAllExcelScientificExpression(String line) throws ParseException {
216 		Matcher matcher = scientificExpressionPattern.matcher(line);
217 
218 		StringBuffer sb = new StringBuffer();
219 		while (matcher.find()) {
220 			// Get the first part
221 			String firstPartString = matcher.group(1);
222 			boolean hasInverseDecimalSeparator = false;
223 			if (firstPartString.indexOf(inverseDecimalSeparator) != -1) {
224 				firstPartString = firstPartString.replace(inverseDecimalSeparator, decimalSeparator);
225 				hasInverseDecimalSeparator = true;
226 			}
227 			double value = numberFormat.parse(firstPartString).doubleValue();
228 
229 			String signString = matcher.group(2);
230 			int sign = 1;
231 			if ("e-".equalsIgnoreCase(signString)) {
232 				sign = -1;
233 			}
234 
235 			// Get the exposant value
236 			String exposantString = matcher.group(3);
237 			int exposant = Integer.parseInt(exposantString);
238 
239 			// Compute the value
240 			String valueString = decimalFormat.format(Math.pow(10, sign * exposant) * value);
241 			//valueString = String.valueOf(value);
242 			if (hasInverseDecimalSeparator) {
243 				valueString = valueString.replace(decimalSeparator, inverseDecimalSeparator);
244 			}
245 
246 			matcher.appendReplacement(sb, valueString);
247 		}
248 		matcher.appendTail(sb);
249 
250 		return sb.toString();
251 	}
252 
253 
254 	public static File getNewTemporaryFile(File sourceFile) {
255 		// Generate a temp file name
256 		File destFile = null;
257 		boolean tmpFileExists = true;
258 		int tempIndex = 0;
259 		while (tmpFileExists) {
260 			destFile = new File(sourceFile.getParent(), getNameWithoutExtension(sourceFile) + ".tmp" + tempIndex++);
261 			tmpFileExists = destFile.exists();
262 		}
263 
264 		return destFile;
265 	}
266 	
267 	public static void removeEmptyLines(File sourceFile) throws IOException {
268 		// Generate a temp file name
269 		File destFile = getNewTemporaryFile(sourceFile);
270 
271 		// Do the replace all
272 		removeEmptyLines(sourceFile, destFile);
273 
274 		// Override the source file content
275 		sourceFile.delete();
276 		destFile.renameTo(sourceFile);
277 	}
278 	
279 	public static void removeEmptyLines(File sourceFile, File destFile) throws IOException {
280 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
281 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
282 
283 		String line = null;
284 		while ((line = reader.readLine()) != null) {
285 			if (StringUtils.isBlank(line) == false) {
286 				writer.write(line);
287 				writer.newLine();
288 			}
289 		}
290 
291 		reader.close();
292 		writer.close();
293 	}
294 
295 
296 	public static void appendLines(File sourceFile, File destFile, String... lines) throws IOException {
297 		checkExists(sourceFile);
298 
299 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
300 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
301 
302 		// Copy lines from file
303 		String line;
304 		while ((line = reader.readLine()) != null) {
305 			writer.write(line);
306 			writer.newLine();
307 		}
308 
309 		// Insert lines at end
310 		for (String endLine : lines) {
311 			writer.write(endLine);
312 			writer.newLine();
313 		}
314 
315 		reader.close();
316 		writer.close();
317 	}
318 
319 	public static void prependLines(File sourceFile, File destFile, String... lines) throws IOException {
320 		checkExists(sourceFile);
321 
322 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
323 		BufferedWriter writer = new BufferedWriter(new FileWriter(destFile));
324 
325 		// Insert lines at beginning
326 		for (String prependLine : lines) {
327 			writer.write(prependLine);
328 			writer.newLine();
329 		}
330 
331 		// Copy lines from file
332 		String line;
333 		while ((line = reader.readLine()) != null) {
334 			writer.write(line);
335 			writer.newLine();
336 		}
337 
338 		reader.close();
339 		writer.close();
340 	}
341 
342 	public static Collection<String> readLines(File sourceFile, int lineCount) throws IOException {
343 		checkExists(sourceFile);
344 
345 		BufferedReader reader = new BufferedReader(new FileReader(sourceFile));
346 
347 		List<String> result = Lists.newArrayList();
348 
349 		// Copy lines from file
350 		String line;
351 		int counter = 0;
352 		while ((line = reader.readLine()) != null && (counter < lineCount || lineCount == -1)) {
353 			result.add(line);
354 			counter++;
355 		}
356 
357 		reader.close();
358 		return result;
359 	}
360 
361 	public static boolean isEmpty(File file) {
362 		return file == null || !file.exists() || file.length() == 0;
363 	}
364 
365 
366 	public static void deleteFiles(File folder, String fileNameRegex) {
367 		for (String filename : folder.list()) {
368 			if (filename.matches(fileNameRegex)) {
369 				File fileTodelete = new File(folder, filename);
370 				fileTodelete.delete();
371 			}
372 		}
373 	}
374 
375 	public static boolean deleteQuietly(File file) {
376 		return org.apache.commons.io.FileUtils.deleteQuietly(file);
377 	}
378 
379 	public static void deleteTemporaryFiles(File file) {
380 		deleteTemporaryFiles(file, TEMPORARY_FILE_DEFAULT_EXTENSION);
381 	}
382 
383 	public static void deleteTemporaryFiles(File file, String suffix) {
384 		Preconditions.checkNotNull(file);
385 
386 		File folder = file.getParentFile();
387 		String regexp = (getNameWithoutExtension(file) + suffix)
388 				// Protect '.' char
389 				.replaceAll("[.]", "[.]")
390 				// add counter parttern
391 				+ "[0-9]+";
392 		deleteFiles(folder, regexp);
393 	}
394 
395 	public static void copyFile(File srcFile, File dstFile) throws IOException {
396 		org.apache.commons.io.FileUtils.copyFile(srcFile, dstFile);
397 	}
398 
399 	public static String getNameWithoutExtension(File srcFile) {
400 		return com.google.common.io.Files.getNameWithoutExtension(srcFile.getName());
401 	}
402 
403 	public static String getExtension(File srcFile) {
404 		return com.google.common.io.Files.getFileExtension(srcFile.getName());
405 	}
406 
407 	/**
408 	 * <p>cleanDirectory.</p>
409 	 *
410 	 * @param directory   a {@link File} object.
411 	 * @param failMessage a {@link String} object.
412 	 */
413 	public static void cleanDirectory(File directory, String failMessage) {
414 
415 		Path path = directory.toPath();
416 		int nbAttempt = 0;
417 		IOException lastException = null;
418 		while (isDirectoryNotEmpty(path) && nbAttempt < 10) {
419 			nbAttempt++;
420 			try {
421 				java.nio.file.Files.walkFileTree(path, new RecursiveDeleteFileVisitor(path));
422 				lastException = null;
423 			} catch (NoSuchFileException ignored) {
424 			} catch (AccessDeniedException ade) {
425 				if (java.nio.file.Files.exists(Paths.get(ade.getFile()))) {
426 					lastException = ade;
427 					break;
428 				}
429 			} catch (IOException e) {
430 				lastException = e;
431 				// wait a while
432 				try {
433 					Thread.sleep(500);
434 				} catch (InterruptedException ignored) {
435 				}
436 			}
437 		}
438 		if (lastException != null) {
439 			throw new SumarisTechnicalException(failMessage, lastException);
440 		}
441 		if (log.isWarnEnabled() && nbAttempt > 1) {
442 			log.warn(String.format("cleaning the directory '%s' successful after %d attempts", directory.getAbsolutePath(), nbAttempt));
443 		}
444 	}
445 
446 	private static boolean isDirectoryNotEmpty(final Path path) {
447 		if (!java.nio.file.Files.isDirectory(path)) {
448 			return false;
449 		}
450 		try {
451 			try (DirectoryStream<Path> dirStream = java.nio.file.Files.newDirectoryStream(path)) {
452 				return dirStream.iterator().hasNext();
453 			}
454 		} catch (IOException e) {
455 			return false;
456 		}
457 	}
458 
459 	private static class RecursiveDeleteFileVisitor extends SimpleFileVisitor<Path> {
460 
461 		private final Path root;
462 
463 		private RecursiveDeleteFileVisitor(Path root) {
464 			this.root = root;
465 		}
466 
467 		@Override
468 		public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
469 
470 			// delete the file
471 			java.nio.file.Files.deleteIfExists(file);
472 			return FileVisitResult.CONTINUE;
473 		}
474 
475 		@Override
476 		public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
477 
478 			// don't delete the root
479 			if (dir == root) {
480 				return FileVisitResult.TERMINATE;
481 			}
482 
483 			// delete the directory
484 			java.nio.file.Files.deleteIfExists(dir);
485 			return FileVisitResult.CONTINUE;
486 		}
487 
488 	}
489 
490 	/**
491 	 * Charge le fichier spécifié par le chemin physique ou comme resource classpath.
492 	 * <br/>Cette méthode regarde d'abord si le chemin spécifié correspond à un chemin physique de fichier sur le disque dur,
493 	 * <br/>et dans le cas contraire tente de charger la ressource correspondante à partir du resource loader spécifié.
494 	 *
495 	 * @param path        	 Le chemin physique ou de package du fichier recherché.
496 	 * @param resourceLoader La resourceLoader Spring à utiliser pour charger le fichier si le chemin
497 	 *                    	 correspond à une resource de classpath.
498 	 * @return Un objet InputStream sur le fichier recherché.
499 	 * @throws SumarisTechnicalException if error
500 	 */
501 	public static InputStream getInputStream(String path, ResourceLoader resourceLoader) throws SumarisTechnicalException {
502 		InputStream is = null;
503 
504 		File f = new File(path);
505 		if (f.exists()) {
506 			try {
507 				is = new FileInputStream(f);
508 			} catch (FileNotFoundException e) {
509 				is = null;
510 			}
511 		} else if (resourceLoader != null) {
512 			Resource r = resourceLoader.getResource(ResourceLoader.CLASSPATH_URL_PREFIX + path);
513 			if (r.exists()) {
514 				try {
515 					is = r.getInputStream();
516 				} catch (IOException e) {
517 					is = null;
518 				}
519 			}
520 		}
521 		if (is == null) {
522 			throw new SumarisTechnicalException(String.format("Resource not found '%s'", path));
523 		}
524 
525 		return is;
526 	}
527 	/**
528 	 * Charge le fichier spécifié par le chemin physique ou de package en paramètre.
529 	 * <br/>Cette méthode regarde d'abord si le chemin spécifié correspond à un chemin physique de fichier sur le disque dur,
530 	 * <br/>et dans le cas contraire tente de charger la ressource correspondante à partir su classloader spécifié.
531 	 *
532 	 * @param path        Le chemin physique ou de package du fichier recherché.
533 	 * @param classLoader La classloader utilisé pour charger le fichier si le chemin
534 	 *                    correspond à un chemin de package.
535 	 * @return Un objet InputStream sur le fichier recherché.
536 	 * @throws SumarisTechnicalException if error
537 	 */
538 	public static InputStream getInputStream(String path, ClassLoader classLoader) throws SumarisTechnicalException {
539 		InputStream is = null;
540 
541 		File f = new File(path);
542 		if (f.exists()) {
543 			try {
544 				is = new FileInputStream(f);
545 			} catch (FileNotFoundException e) {
546 				is = null;
547 			}
548 		} else if (classLoader != null) {
549 			is = classLoader.getResourceAsStream(path);
550 		}
551 		if (is == null) {
552 			throw new SumarisTechnicalException(String.format("File not found '%s'", path));
553 		}
554 
555 		return is;
556 	}
557 
558 	/**
559 	 * Crée un fichier temporaire, avec l'extension par défaut
560 	 * @param prefix
561 	 * @param directory
562 	 * @return
563 	 * @throws IOException
564 	 */
565 	public static File createTempFile(String prefix, File directory) throws IOException {
566 		if (directory != null && !directory.exists()) {
567 			directory.mkdirs();
568 		}
569 
570 		return File.createTempFile(
571 				prefix,
572 				TEMPORARY_FILE_DEFAULT_EXTENSION,
573 				directory);
574 	}
575 }