1 package fr.ifremer.dali.service.synchro;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26 import com.google.common.collect.ArrayListMultimap;
27 import com.google.common.collect.Lists;
28 import com.google.common.collect.Multimap;
29 import fr.ifremer.common.synchro.config.SynchroConfiguration;
30 import fr.ifremer.common.synchro.service.RejectedRow;
31 import fr.ifremer.dali.dao.data.survey.DaliSurveyDao;
32 import fr.ifremer.dali.dao.technical.Daos;
33 import fr.ifremer.dali.dto.DaliBeanFactory;
34 import fr.ifremer.dali.dto.DaliBeans;
35 import fr.ifremer.dali.dto.system.synchronization.SynchroChangesDTO;
36 import fr.ifremer.dali.dto.system.synchronization.SynchroRowDTO;
37 import fr.ifremer.dali.dto.system.synchronization.SynchroTableDTO;
38 import fr.ifremer.dali.service.DaliTechnicalException;
39 import fr.ifremer.quadrige3.core.ProgressionCoreModel;
40 import fr.ifremer.quadrige3.core.config.QuadrigeConfiguration;
41 import fr.ifremer.quadrige3.core.dao.referential.ReferentialJdbcDao;
42 import fr.ifremer.quadrige3.core.dao.referential.ReferentialJdbcDaoImpl;
43 import fr.ifremer.quadrige3.core.dao.technical.Assert;
44 import fr.ifremer.quadrige3.core.dao.technical.decorator.Decorator;
45 import fr.ifremer.quadrige3.core.exception.QuadrigeTechnicalException;
46 import fr.ifremer.quadrige3.core.service.decorator.DecoratorService;
47 import fr.ifremer.quadrige3.core.vo.data.survey.LightSurveyVO;
48 import fr.ifremer.quadrige3.synchro.meta.administration.ProgramStrategySynchroTables;
49 import fr.ifremer.quadrige3.synchro.meta.data.DataSynchroTables;
50 import fr.ifremer.quadrige3.synchro.meta.referential.ReferentialSynchroTables;
51 import fr.ifremer.quadrige3.synchro.meta.system.RuleSynchroTables;
52 import fr.ifremer.quadrige3.synchro.service.client.SynchroClientInternalService;
53 import fr.ifremer.quadrige3.synchro.service.client.vo.SynchroOperationType;
54 import fr.ifremer.quadrige3.synchro.vo.SynchroChangesVO;
55 import fr.ifremer.quadrige3.synchro.vo.SynchroImportContextVO;
56 import org.apache.commons.collections4.CollectionUtils;
57 import org.apache.commons.io.FileUtils;
58 import org.apache.commons.lang3.StringUtils;
59 import org.apache.commons.logging.Log;
60 import org.apache.commons.logging.LogFactory;
61 import org.springframework.beans.factory.annotation.Autowired;
62 import org.springframework.stereotype.Service;
63
64 import javax.annotation.Resource;
65 import java.io.File;
66 import java.sql.SQLException;
67 import java.util.*;
68
69 import static org.nuiton.i18n.I18n.t;
70
71
72
73
74 @Service("daliSynchroClientService")
75 public class SynchroClientServiceImpl
76 extends fr.ifremer.quadrige3.synchro.service.client.SynchroClientServiceImpl
77 implements SynchroClientService {
78
79 private static final Log log = LogFactory.getLog(SynchroClientServiceImpl.class);
80
81 @Resource(name = "daliSurveyDao")
82 private DaliSurveyDao surveyDao;
83
84 @Resource(name = "referentialJdbcDao")
85 private ReferentialJdbcDao targetReferentialJdbcDao;
86
87 @Resource(name = "synchroClientService")
88 private SynchroClientInternalService synchroClientInternalService;
89
90
91
92
93
94
95
96
97 @Autowired
98 public SynchroClientServiceImpl(QuadrigeConfiguration config, SynchroConfiguration synchroConfig, fr.ifremer.dali.decorator.DecoratorService decoratorService) {
99 super(config, synchroConfig, decoratorService);
100 }
101
102
103
104
105
106
107 @Override
108 public SynchroChangesDTO getImportFileChangesAsDTO(
109 int userId,
110 File dbZipFile,
111 SynchroImportContextVO importContext,
112 ProgressionCoreModel progressionModel,
113 int progressionModelMaxCount) {
114
115 SynchroChangesDTO insertUpdateChanges;
116 SynchroChangesDTO referentialDeleteChanges;
117
118
119 {
120 SynchroChangesVO changes = synchroClientInternalService.getImportFileInsertAndUpdateChangesTransactional(userId,
121 dbZipFile,
122 importContext,
123 progressionModel,
124 progressionModelMaxCount,
125 true );
126
127
128 String tempDbJdbcUrl = Daos.getUrl(changes.getConnectionProperties());
129 File tempDirectory = new File(Daos.getDbDirectoryFromJdbcUrl(tempDbJdbcUrl));
130
131 insertUpdateChanges = toSynchroChangesDTO(changes.getConnectionProperties(), dbZipFile, changes);
132
133
134 FileUtils.deleteQuietly(tempDirectory.getParentFile());
135 }
136
137
138 {
139 SynchroChangesVO changes = synchroClientInternalService.getImportFileReferentialDeleteChangesTransactional(userId,
140 dbZipFile,
141 importContext,
142 progressionModel,
143 progressionModelMaxCount,
144 true );
145
146
147 String tempDbJdbcUrl = Daos.getUrl(changes.getConnectionProperties());
148 File tempDirectory = new File(Daos.getDbDirectoryFromJdbcUrl(tempDbJdbcUrl));
149
150 referentialDeleteChanges = toSynchroChangesDTO(changes.getConnectionProperties(), dbZipFile, changes);
151
152
153 FileUtils.deleteQuietly(tempDirectory.getParentFile());
154 }
155
156
157 return mergeChanges(insertUpdateChanges, referentialDeleteChanges);
158 }
159
160 private SynchroChangesDTO mergeChanges(SynchroChangesDTO changes, SynchroChangesDTO changesToAdd) {
161 Assert.notNull(changes);
162 Assert.notNull(changesToAdd);
163
164 Map<String, SynchroTableDTO> tablesByName = DaliBeans.mapByProperty(changes.getTables(), SynchroTableDTO.PROPERTY_NAME);
165
166 for (SynchroTableDTO tableToAdd: changesToAdd.getTables()) {
167 SynchroTableDTO table = tablesByName.get(tableToAdd.getName());
168 if (table == null) {
169
170 changes.addTables(tableToAdd);
171 } else {
172
173 table.addAllRows(tableToAdd.getRows());
174 }
175 }
176
177 return changes;
178 }
179
180
181 @Override
182 public Multimap<String, String> toPkToIncludesMap(SynchroChangesDTO changes) {
183 Multimap<String, String> result = ArrayListMultimap.create();
184
185 if (changes != null && changes.getTables() != null) {
186 for (SynchroTableDTO table : changes.getTables()) {
187 String tableName = table.getName();
188
189 if (!table.isRowsEmpty()) {
190 for (SynchroRowDTO row : table.getRows()) {
191 String strategyStr = row.getStrategy();
192 if (StringUtils.isNotBlank(strategyStr)) {
193 RejectedRow.ResolveStrategy strategy = RejectedRow.ResolveStrategy.valueOf(strategyStr);
194 if (strategy == RejectedRow.ResolveStrategy.UPDATE
195 || strategy == RejectedRow.ResolveStrategy.DUPLICATE) {
196 result.put(tableName, row.getPkStr());
197 }
198 }
199 }
200 }
201 }
202 }
203 return result;
204 }
205
206
207 @Override
208 public boolean hasProgramOrRulesChanges(SynchroChangesDTO synchroChanges) {
209
210 if (synchroChanges == null || synchroChanges.isTablesEmpty()) {
211 return false;
212 }
213
214 for (SynchroTableDTO table : synchroChanges.getTables()) {
215 if (ProgramStrategySynchroTables.tableNames().contains(table.getName().toUpperCase())
216 || RuleSynchroTables.tableNames().contains(table.getName().toUpperCase())) {
217 return true;
218 }
219 }
220 return false;
221 }
222
223
224 @Override
225 public SynchroChangesDTO getReferentialSynchroChangesToApplyFromFile(SynchroChangesDTO synchroChanges) {
226 if (synchroChanges == null || synchroChanges.isTablesEmpty()) {
227 return synchroChanges;
228 }
229
230
231 Set<String> referentialTablesOnly = new LinkedHashSet<>(ReferentialSynchroTables.tableNames());
232 referentialTablesOnly.removeAll(ProgramStrategySynchroTables.tableNames());
233
234 SynchroChangesDTO filteredSynchroChanges = DaliBeans.clone(synchroChanges);
235 filteredSynchroChanges.setTables(new ArrayList<>());
236
237
238 for (SynchroTableDTO table : synchroChanges.getTables()) {
239 String tableName = table.getName().toUpperCase();
240 boolean isProgramStrategy = ProgramStrategySynchroTables.tableNames().contains(tableName);
241 boolean isRule = RuleSynchroTables.tableNames().contains(tableName);
242 boolean isReferential = isProgramStrategy || isRule || referentialTablesOnly.contains(tableName);
243
244 SynchroTableDTO filteredSynchroTable = DaliBeans.clone(table);
245 filteredSynchroTable.setRows(new ArrayList<>());
246 if (!table.isRowsEmpty()) {
247
248 for (SynchroRowDTO row : table.getRows()) {
249
250 boolean addRow = false;
251 switch (SynchroOperationType.valueOf(row.getOperationType().toUpperCase())) {
252
253 case INSERT:
254
255 addRow = isReferential;
256 break;
257 case UPDATE:
258
259 addRow = isProgramStrategy || isRule;
260 break;
261 case DELETE:
262
263 addRow = isProgramStrategy || isRule;
264 break;
265 case DUPLICATE:
266
267 break;
268 }
269
270 if (addRow) {
271 filteredSynchroTable.addRows(row);
272 }
273 }
274 }
275
276 if (!filteredSynchroTable.isRowsEmpty()) {
277 filteredSynchroChanges.addTables(filteredSynchroTable);
278 }
279 }
280
281
282 return filteredSynchroChanges;
283 }
284
285
286 @Override
287 public SynchroChangesDTO getSurveySynchroChangesFromFile(SynchroChangesDTO synchroChanges) {
288 if (synchroChanges == null || synchroChanges.isTablesEmpty()) {
289 return synchroChanges;
290 }
291
292 SynchroChangesDTO filteredSynchroChanges = DaliBeans.clone(synchroChanges);
293 filteredSynchroChanges.setTables(new ArrayList<>());
294
295
296 for (SynchroTableDTO table : synchroChanges.getTables()) {
297 String tableName = table.getName().toUpperCase();
298
299 if (DataSynchroTables.SURVEY.name().equals(tableName)) {
300
301 SynchroTableDTO filteredSynchroTable = DaliBeans.clone(table);
302 filteredSynchroTable.setRows(new ArrayList<>());
303 if (!table.isRowsEmpty()) {
304
305 for (SynchroRowDTO row : table.getRows()) {
306 SynchroOperationType operationType = SynchroOperationType.valueOf(row.getOperationType().toUpperCase());
307 if (operationType == SynchroOperationType.INSERT
308 || operationType == SynchroOperationType.DUPLICATE
309 || operationType == SynchroOperationType.IGNORE) {
310 filteredSynchroTable.addRows(row);
311 }
312 }
313 }
314
315
316 if (!filteredSynchroTable.isRowsEmpty()) {
317 filteredSynchroChanges.addTables(filteredSynchroTable);
318 }
319 }
320 }
321
322
323 return filteredSynchroChanges;
324 }
325
326
327
328
329
330
331
332
333
334
335
336 private SynchroChangesDTO toSynchroChangesDTO(Properties changesConnectionProperties, File file, SynchroChangesVO source) {
337 SynchroChangesDTO target = DaliBeanFactory.newSynchroChangesDTO();
338
339
340 target.setFile(file);
341
342 if (source != null && !source.isEmpty()) {
343
344 try {
345 ReferentialJdbcDao changesReferentialJdbcDao = new ReferentialJdbcDaoImpl(changesConnectionProperties);
346
347 List<SynchroTableDTO> tables = Lists.newArrayListWithCapacity(source.getTableNames().size());
348 for (String tableName : source.getTableNames()) {
349
350 SynchroTableDTO table = DaliBeanFactory.newSynchroTableDTO();
351 table.setName(tableName);
352
353
354 List<SynchroRowDTO> rows = Lists.newArrayList();
355 table.setRows(rows);
356
357
358 rows.addAll(toSynchroRowDTOs(changesReferentialJdbcDao, tableName, source.getInserts(tableName), SynchroOperationType.INSERT));
359
360
361 rows.addAll(toSynchroRowDTOs(changesReferentialJdbcDao, tableName, source.getUpdates(tableName), SynchroOperationType.UPDATE));
362
363
364 rows.addAll(toSynchroRowDTOs(changesReferentialJdbcDao, tableName, source.getDeletes(tableName), SynchroOperationType.DELETE));
365
366
367 if (tableName.equalsIgnoreCase(DataSynchroTables.SURVEY.name())) {
368 String rejectedRows = source.getRejectedRows(tableName);
369 rows.addAll(surveyRejectsToSynchroRowDTOs(rejectedRows, SynchroOperationType.DUPLICATE));
370 rows.addAll(surveyRejectsToSynchroRowDTOs(rejectedRows, SynchroOperationType.IGNORE));
371 }
372
373
374 if (!table.isRowsEmpty()) {
375 tables.add(table);
376 }
377 }
378 target.addAllTables(tables);
379 } finally {
380 try {
381 Daos.shutdownDatabase(changesConnectionProperties);
382 } catch (SQLException ignored) {
383
384 log.warn(t("quadrige3.error.synchro.import.shutdown"));
385 }
386 }
387 }
388
389 return target;
390 }
391
392
393
394
395
396
397
398
399
400
401 private List<SynchroRowDTO> toSynchroRowDTOs(ReferentialJdbcDao changesReferentialJdbcDao, String tableName, Collection<String> pks, SynchroOperationType operationType) {
402 List<SynchroRowDTO> result = Lists.newArrayList();
403 if (CollectionUtils.isEmpty(pks)) {
404 return result;
405 }
406
407 DecoratorService decoratorService = getDecoratorService();
408 Decorator<?> decorator = null;
409 boolean useDecorator = true;
410
411 for (String pk : pks) {
412 String name = null;
413
414 if (useDecorator) {
415 Object vo = null;
416 try {
417
418 vo = changesReferentialJdbcDao.getVOByTableNameAndPk(tableName, pk);
419 if (vo == null) {
420
421 vo = targetReferentialJdbcDao.getVOByTableNameAndPk(tableName, pk);
422 }
423 } catch (QuadrigeTechnicalException ignored) {
424 }
425
426
427 if (decorator == null) {
428 if (vo == null) {
429 log.warn(String.format("[%s] Unable to load object with [pk=%s]. Check if referentialJdbcDao.getVOByTableNameAndPk() is well implemented for this table.", tableName, pk));
430 useDecorator = false;
431 } else {
432 decorator = decoratorService.getDecorator(vo);
433 if (decorator == null) {
434 log.warn(String.format("[%s] Unable to find decorator for class [%s]. Please add a default decorator.", tableName, vo.getClass().getSimpleName()));
435 useDecorator = false;
436 }
437 }
438 }
439 if (useDecorator) {
440 name = decorator.toString(vo);
441 }
442 }
443
444
445 if (!useDecorator) {
446 name = String.format("%s (%s)", tableName, pk);
447 }
448
449
450 SynchroRowDTO row = DaliBeanFactory.newSynchroRowDTO();
451 row.setName(name);
452 row.setOperationType(operationType.name());
453 row.setPkStr(pk);
454
455
456 row.setStrategy(null);
457 result.add(row);
458 }
459
460 return result;
461 }
462
463
464
465
466
467
468
469
470 private List<SynchroRowDTO> surveyRejectsToSynchroRowDTOs(String rejectedRows, SynchroOperationType operationType) {
471 List<SynchroRowDTO> result = Lists.newArrayList();
472 if (StringUtils.isBlank(rejectedRows)) {
473 return result;
474 }
475
476 List<RejectedRow> rejects = RejectedRow.parseFromString(rejectedRows);
477 for (RejectedRow reject : rejects) {
478
479 if (operationType == SynchroOperationType.DUPLICATE) {
480 if (reject.cause == RejectedRow.Cause.DUPLICATE_KEY
481 && reject.targetPkStr != null) {
482 int targetSurveyId = Integer.parseInt(reject.targetPkStr);
483 LightSurveyVO surveyVO = surveyDao.getLightSurveyById(targetSurveyId);
484 if (surveyVO == null) {
485 throw new DaliTechnicalException(String.format("Could not load survey with id [%s]", targetSurveyId));
486 }
487 String name = decorate(surveyVO);
488
489
490 SynchroRowDTO row = DaliBeanFactory.newSynchroRowDTO();
491 row.setName(name);
492 row.setOperationType(operationType.name());
493 row.setPkStr(reject.pkStr);
494
495
496 row.setStrategy(null);
497 result.add(row);
498 }
499 }
500
501 else if (operationType == SynchroOperationType.IGNORE) {
502 if (reject.cause == RejectedRow.Cause.MISSING_FOREIGN_KEY
503 && reject.targetPkStr != null
504 && reject.fkColumnName != null
505 && reject.targetFkValue != null) {
506
507
508 SynchroRowDTO row = DaliBeanFactory.newSynchroRowDTO();
509 row.setName(reject.targetFkValue);
510 row.setOperationType(operationType.name());
511 row.setPkStr(reject.pkStr);
512
513
514 row.setStrategy(null);
515 result.add(row);
516 }
517 }
518 }
519
520 return result;
521 }
522
523 }