001/* 002 * Licensed under the Apache License, Version 2.0 (the "License"); 003 * you may not use this file except in compliance with the License. 004 * You may obtain a copy of the License at 005 * 006 * http://www.apache.org/licenses/LICENSE-2.0 007 * 008 * Unless required by applicable law or agreed to in writing, software 009 * distributed under the License is distributed on an "AS IS" BASIS, 010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 011 * See the License for the specific language governing permissions and 012 * limitations under the License. 013 */ 014package org.gbif.api.model.pipelines; 015 016import org.gbif.api.model.pipelines.PipelinesWorkflow.Graph; 017 018import java.util.Arrays; 019import java.util.HashSet; 020import java.util.List; 021import java.util.Set; 022import java.util.function.BiPredicate; 023 024import org.junit.jupiter.api.Assertions; 025import org.junit.jupiter.api.Test; 026 027import static org.gbif.api.model.pipelines.StepType.*; 028 029 030public class PipelinesWorkflowTest { 031 032 private final BiPredicate<List<Graph<StepType>.Edge>, StepType> containsFn = 033 (l, t) -> l.stream().anyMatch(x -> x.getNode().equals(t)); 034 035 @Test 036 public void findAllStepsToRunTest() { 037 Set<StepType> types = new HashSet<>( 038 Arrays.asList(VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW, INTERPRETED_TO_INDEX)); 039 040 Set<StepType> allNodesFor = PipelinesWorkflow.getOccurrenceWorkflow().getAllNodesFor(types); 041 042 Assertions.assertEquals(5, allNodesFor.size()); 043 Assertions.assertTrue(allNodesFor.containsAll( 044 Arrays.asList(VERBATIM_TO_IDENTIFIER, VERBATIM_TO_INTERPRETED, HDFS_VIEW, INTERPRETED_TO_INDEX, FRAGMENTER))); 045 } 046 047 @Test 048 public void findAllStepsToRunSameLevelTest() { 049 Set<StepType> types = 050 new HashSet<>(Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED)); 051 052 Set<StepType> allNodesFor = PipelinesWorkflow.getEventOccurrenceWorkflow().getAllNodesFor(types); 053 054 Assertions.assertEquals(5, allNodesFor.size()); 055 Assertions.assertTrue(allNodesFor.containsAll( 056 Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_HDFS_VIEW, 057 EVENTS_INTERPRETED_TO_INDEX))); 058 } 059 060 @Test 061 public void findOneRootStepsToRunTest() { 062 Set<StepType> types = new HashSet<>( 063 Arrays.asList(DWCA_TO_VERBATIM, VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW, 064 INTERPRETED_TO_INDEX)); 065 066 Set<StepType> rootNodes = PipelinesWorkflow.getOccurrenceWorkflow().getRootNodesFor(types); 067 068 Assertions.assertEquals(1, rootNodes.size()); 069 Assertions.assertTrue(rootNodes.contains(DWCA_TO_VERBATIM)); 070 } 071 072 @Test 073 public void findMainRootStepsToRunTest() { 074 Set<StepType> types = new HashSet<>( 075 Arrays.asList(VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW, INTERPRETED_TO_INDEX)); 076 077 Set<StepType> rootNodes = PipelinesWorkflow.getOccurrenceWorkflow().getRootNodesFor(types); 078 079 Assertions.assertEquals(1, rootNodes.size()); 080 Assertions.assertTrue(rootNodes.contains(VERBATIM_TO_IDENTIFIER)); 081 } 082 083 @Test 084 public void findRootStepsToRunSameLevelTest() { 085 Set<StepType> types = 086 new HashSet<>(Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_HDFS_VIEW, 087 EVENTS_INTERPRETED_TO_INDEX)); 088 089 Set<StepType> rootNodes = PipelinesWorkflow.getEventOccurrenceWorkflow().getRootNodesFor(types); 090 091 Assertions.assertEquals(3, rootNodes.size()); 092 Assertions.assertTrue(rootNodes.containsAll( 093 Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED))); 094 } 095 096 @Test 097 public void eventOccurrenceWorkflowTest() { 098 099 Graph<StepType> wf = PipelinesWorkflow.getEventOccurrenceWorkflow(); 100 101 Assertions.assertEquals(9, wf.getNodesQuantity()); 102 103 Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM)); 104 Assertions.assertEquals(2, wf.getLevel(VERBATIM_TO_IDENTIFIER)); 105 Assertions.assertEquals(3, wf.getLevel(VERBATIM_TO_INTERPRETED)); 106 Assertions.assertEquals(4, wf.getLevel(INTERPRETED_TO_INDEX)); 107 Assertions.assertEquals(4, wf.getLevel(HDFS_VIEW)); 108 Assertions.assertEquals(4, wf.getLevel(FRAGMENTER)); 109 Assertions.assertEquals(4, wf.getLevel(EVENTS_VERBATIM_TO_INTERPRETED)); 110 Assertions.assertEquals(5, wf.getLevel(EVENTS_INTERPRETED_TO_INDEX)); 111 Assertions.assertEquals(5, wf.getLevel(EVENTS_HDFS_VIEW)); 112 113 // DWCA_TO_VERBATIM -> VERBATIM_TO_IDENTIFIER 114 List<Graph<StepType>.Edge> dwcaToVerbatimEdges = wf.getNodeEdges(DWCA_TO_VERBATIM); 115 Assertions.assertEquals(1, dwcaToVerbatimEdges.size()); 116 Assertions.assertEquals(VERBATIM_TO_IDENTIFIER, dwcaToVerbatimEdges.get(0).getNode()); 117 118 // VERBATIM_TO_IDENTIFIER -> VERBATIM_TO_INTERPRETED 119 List<Graph<StepType>.Edge> verbatimToIdentifierEdges = wf.getNodeEdges(VERBATIM_TO_IDENTIFIER); 120 Assertions.assertEquals(1, verbatimToIdentifierEdges.size()); 121 Assertions.assertEquals(VERBATIM_TO_INTERPRETED, verbatimToIdentifierEdges.get(0).getNode()); 122 123 // VERBATIM_TO_INTERPRETED -> INTERPRETED_TO_INDEX & HDFS_VIEW & FRAGMENTER & EVENTS_VERBATIM_TO_INTERPRETED 124 List<Graph<StepType>.Edge> verbatimToInterpretedEdges = wf.getNodeEdges(VERBATIM_TO_INTERPRETED); 125 Assertions.assertEquals(4, verbatimToInterpretedEdges.size()); 126 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, INTERPRETED_TO_INDEX)); 127 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, HDFS_VIEW)); 128 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, FRAGMENTER)); 129 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, EVENTS_VERBATIM_TO_INTERPRETED)); 130 131 // INTERPRETED_TO_INDEX -> 0 132 List<Graph<StepType>.Edge> interpretedToIndexEdges = wf.getNodeEdges(INTERPRETED_TO_INDEX); 133 Assertions.assertEquals(0, interpretedToIndexEdges.size()); 134 135 // HDFS_VIEW -> 0 136 List<Graph<StepType>.Edge> hdfsViewEdges = wf.getNodeEdges(HDFS_VIEW); 137 Assertions.assertEquals(0, hdfsViewEdges.size()); 138 139 // FRAGMENTER -> 0 140 List<Graph<StepType>.Edge> fragmenterEdges = wf.getNodeEdges(FRAGMENTER); 141 Assertions.assertEquals(0, fragmenterEdges.size()); 142 143 // EVENTS_VERBATIM_TO_INTERPRETED -> EVENTS_INTERPRETED_TO_INDEX & EVENTS_HDFS_VIEW 144 List<Graph<StepType>.Edge> interpretationNodeEdges = wf.getNodeEdges(EVENTS_VERBATIM_TO_INTERPRETED); 145 Assertions.assertEquals(2, interpretationNodeEdges.size()); 146 Assertions.assertTrue(containsFn.test(interpretationNodeEdges, EVENTS_INTERPRETED_TO_INDEX)); 147 Assertions.assertTrue(containsFn.test(interpretationNodeEdges, EVENTS_HDFS_VIEW)); 148 149 // EVENTS_INTERPRETED_TO_INDEX -> 0 150 List<Graph<StepType>.Edge> eventsInterpretedToIndexEdges = wf.getNodeEdges(EVENTS_INTERPRETED_TO_INDEX); 151 Assertions.assertEquals(0, eventsInterpretedToIndexEdges.size()); 152 153 // EVENTS_HDFS_VIEW -> 0 154 List<Graph<StepType>.Edge> eventsHdfsViewEdges = wf.getNodeEdges(EVENTS_HDFS_VIEW); 155 Assertions.assertEquals(0, eventsHdfsViewEdges.size()); 156 } 157 158 @Test 159 public void eventOnlyWorkflowTest() { 160 161 Graph<StepType> wf = PipelinesWorkflow.getEventWorkflow(); 162 163 Assertions.assertEquals(4, wf.getNodesQuantity()); 164 165 Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM)); 166 Assertions.assertEquals(2, wf.getLevel(EVENTS_VERBATIM_TO_INTERPRETED)); 167 Assertions.assertEquals(3, wf.getLevel(EVENTS_INTERPRETED_TO_INDEX)); 168 Assertions.assertEquals(3, wf.getLevel(EVENTS_HDFS_VIEW)); 169 170 // DWCA_TO_VERBATIM -> EVENTS_VERBATIM_TO_INTERPRETED 171 List<Graph<StepType>.Edge> dwcaToVerbatimEdges = wf.getNodeEdges(DWCA_TO_VERBATIM); 172 Assertions.assertEquals(1, dwcaToVerbatimEdges.size()); 173 Assertions.assertEquals(EVENTS_VERBATIM_TO_INTERPRETED, dwcaToVerbatimEdges.get(0).getNode()); 174 175 // EVENTS_VERBATIM_TO_INTERPRETED -> EVENTS_INTERPRETED_TO_INDEX & EVENTS_HDFS_VIEW 176 List<Graph<StepType>.Edge> interpretationNodeEdges = wf.getNodeEdges(EVENTS_VERBATIM_TO_INTERPRETED); 177 Assertions.assertEquals(2, interpretationNodeEdges.size()); 178 Assertions.assertTrue(containsFn.test(interpretationNodeEdges, EVENTS_INTERPRETED_TO_INDEX)); 179 Assertions.assertTrue(containsFn.test(interpretationNodeEdges, EVENTS_HDFS_VIEW)); 180 181 // EVENTS_INTERPRETED_TO_INDEX -> 0 182 List<Graph<StepType>.Edge> eventsInterpretedToIndexEdges = wf.getNodeEdges(EVENTS_INTERPRETED_TO_INDEX); 183 Assertions.assertEquals(0, eventsInterpretedToIndexEdges.size()); 184 185 // EVENTS_HDFS_VIEW -> 0 186 List<Graph<StepType>.Edge> eventsHdfsViewEdges = wf.getNodeEdges(EVENTS_HDFS_VIEW); 187 Assertions.assertEquals(0, eventsHdfsViewEdges.size()); 188 } 189 190 @Test 191 public void occurrenceWorkflowTest() { 192 193 Graph<StepType> wf = PipelinesWorkflow.getOccurrenceWorkflow(); 194 195 Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM)); 196 Assertions.assertEquals(1, wf.getLevel(ABCD_TO_VERBATIM)); 197 Assertions.assertEquals(1, wf.getLevel(XML_TO_VERBATIM)); 198 Assertions.assertEquals(2, wf.getLevel(VERBATIM_TO_IDENTIFIER)); 199 Assertions.assertEquals(3, wf.getLevel(VERBATIM_TO_INTERPRETED)); 200 Assertions.assertEquals(4, wf.getLevel(INTERPRETED_TO_INDEX)); 201 Assertions.assertEquals(4, wf.getLevel(HDFS_VIEW)); 202 Assertions.assertEquals(4, wf.getLevel(FRAGMENTER)); 203 204 Assertions.assertEquals(8, wf.getNodesQuantity()); 205 206 // DWCA_TO_VERBATIM -> VERBATIM_TO_IDENTIFIER 207 List<Graph<StepType>.Edge> dwcaToVerbatimEdges = wf.getNodeEdges(DWCA_TO_VERBATIM); 208 Assertions.assertEquals(1, dwcaToVerbatimEdges.size()); 209 Assertions.assertEquals(VERBATIM_TO_IDENTIFIER, dwcaToVerbatimEdges.get(0).getNode()); 210 211 // ABCD_TO_VERBATIM -> VERBATIM_TO_IDENTIFIER 212 List<Graph<StepType>.Edge> abcdToVerbatimEdges = wf.getNodeEdges(ABCD_TO_VERBATIM); 213 Assertions.assertEquals(1, abcdToVerbatimEdges.size()); 214 Assertions.assertEquals(VERBATIM_TO_IDENTIFIER, abcdToVerbatimEdges.get(0).getNode()); 215 216 // XML_TO_VERBATIM -> VERBATIM_TO_IDENTIFIER 217 List<Graph<StepType>.Edge> xmlToVerbatimEdges = wf.getNodeEdges(XML_TO_VERBATIM); 218 Assertions.assertEquals(1, xmlToVerbatimEdges.size()); 219 Assertions.assertEquals(VERBATIM_TO_IDENTIFIER, xmlToVerbatimEdges.get(0).getNode()); 220 221 // VERBATIM_TO_IDENTIFIER -> VERBATIM_TO_INTERPRETED 222 List<Graph<StepType>.Edge> verbatimToIdentifierEdges = wf.getNodeEdges(VERBATIM_TO_IDENTIFIER); 223 Assertions.assertEquals(1, verbatimToIdentifierEdges.size()); 224 Assertions.assertEquals(VERBATIM_TO_INTERPRETED, verbatimToIdentifierEdges.get(0).getNode()); 225 226 // VERBATIM_TO_INTERPRETED -> INTERPRETED_TO_INDEX & HDFS_VIEW & FRAGMENTER 227 List<Graph<StepType>.Edge> verbatimToInterpretedEdges = wf.getNodeEdges(VERBATIM_TO_INTERPRETED); 228 Assertions.assertEquals(3, verbatimToInterpretedEdges.size()); 229 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, INTERPRETED_TO_INDEX)); 230 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, HDFS_VIEW)); 231 Assertions.assertTrue(containsFn.test(verbatimToInterpretedEdges, FRAGMENTER)); 232 233 // INTERPRETED_TO_INDEX -> 0 234 List<Graph<StepType>.Edge> interpretedToIndexEdges = wf.getNodeEdges(INTERPRETED_TO_INDEX); 235 Assertions.assertEquals(0, interpretedToIndexEdges.size()); 236 237 // HDFS_VIEW -> 0 238 List<Graph<StepType>.Edge> hdfsViewEdges = wf.getNodeEdges(HDFS_VIEW); 239 Assertions.assertEquals(0, hdfsViewEdges.size()); 240 241 // FRAGMENTER -> 0 242 List<Graph<StepType>.Edge> fragmenterEdges = wf.getNodeEdges(FRAGMENTER); 243 Assertions.assertEquals(0, fragmenterEdges.size()); 244 } 245 246 @Test 247 public void validatorWorkflowTest() { 248 249 Graph<StepType> wf = PipelinesWorkflow.getValidatorWorkflow(); 250 251 Assertions.assertEquals(9, wf.getNodesQuantity()); 252 253 Assertions.assertEquals(1, wf.getLevel(VALIDATOR_UPLOAD_ARCHIVE)); 254 Assertions.assertEquals(2, wf.getLevel(VALIDATOR_VALIDATE_ARCHIVE)); 255 Assertions.assertEquals(3, wf.getLevel(VALIDATOR_DWCA_TO_VERBATIM)); 256 Assertions.assertEquals(3, wf.getLevel(VALIDATOR_ABCD_TO_VERBATIM)); 257 Assertions.assertEquals(3, wf.getLevel(VALIDATOR_XML_TO_VERBATIM)); 258 Assertions.assertEquals(3, wf.getLevel(VALIDATOR_TABULAR_TO_VERBATIM)); 259 Assertions.assertEquals(4, wf.getLevel(VALIDATOR_VERBATIM_TO_INTERPRETED)); 260 Assertions.assertEquals(5, wf.getLevel(VALIDATOR_INTERPRETED_TO_INDEX)); 261 Assertions.assertEquals(6, wf.getLevel(VALIDATOR_COLLECT_METRICS)); 262 263 // VALIDATOR_UPLOAD_ARCHIVE -> VALIDATOR_VALIDATE_ARCHIVE 264 List<Graph<StepType>.Edge> validatorUploadArchiveEdges = wf.getNodeEdges(VALIDATOR_UPLOAD_ARCHIVE); 265 Assertions.assertEquals(1, validatorUploadArchiveEdges.size()); 266 Assertions.assertEquals(VALIDATOR_VALIDATE_ARCHIVE, validatorUploadArchiveEdges.get(0).getNode()); 267 268 // VALIDATOR_VALIDATE_ARCHIVE -> VALIDATOR_DWCA_TO_VERBATIM & VALIDATOR_ABCD_TO_VERBATIM & VALIDATOR_XML_TO_VERBATIM & VALIDATOR_TABULAR_TO_VERBATIM 269 List<Graph<StepType>.Edge> validatorValidateArchiveEdges = wf.getNodeEdges(VALIDATOR_VALIDATE_ARCHIVE); 270 Assertions.assertEquals(4, validatorValidateArchiveEdges.size()); 271 Assertions.assertTrue(containsFn.test(validatorValidateArchiveEdges, VALIDATOR_DWCA_TO_VERBATIM)); 272 Assertions.assertTrue(containsFn.test(validatorValidateArchiveEdges, VALIDATOR_ABCD_TO_VERBATIM)); 273 Assertions.assertTrue(containsFn.test(validatorValidateArchiveEdges, VALIDATOR_XML_TO_VERBATIM)); 274 Assertions.assertTrue(containsFn.test(validatorValidateArchiveEdges, VALIDATOR_TABULAR_TO_VERBATIM)); 275 276 // VALIDATOR_DWCA_TO_VERBATIM -> VALIDATOR_VERBATIM_TO_INTERPRETED 277 List<Graph<StepType>.Edge> dwcaToVerbatimEdges = wf.getNodeEdges(VALIDATOR_DWCA_TO_VERBATIM); 278 Assertions.assertEquals(1, dwcaToVerbatimEdges.size()); 279 Assertions.assertEquals(VALIDATOR_VERBATIM_TO_INTERPRETED, dwcaToVerbatimEdges.get(0).getNode()); 280 281 // VALIDATOR_ABCD_TO_VERBATIM -> VALIDATOR_VERBATIM_TO_INTERPRETED 282 List<Graph<StepType>.Edge> abcdToVerbatimEdges = wf.getNodeEdges(VALIDATOR_ABCD_TO_VERBATIM); 283 Assertions.assertEquals(1, abcdToVerbatimEdges.size()); 284 Assertions.assertEquals(VALIDATOR_VERBATIM_TO_INTERPRETED, abcdToVerbatimEdges.get(0).getNode()); 285 286 // VALIDATOR_XML_TO_VERBATIM -> VALIDATOR_VERBATIM_TO_INTERPRETED 287 List<Graph<StepType>.Edge> xmlToVerbatimEdges = wf.getNodeEdges(VALIDATOR_XML_TO_VERBATIM); 288 Assertions.assertEquals(1, xmlToVerbatimEdges.size()); 289 Assertions.assertEquals(VALIDATOR_VERBATIM_TO_INTERPRETED, xmlToVerbatimEdges.get(0).getNode()); 290 291 // VALIDATOR_TABULAR_TO_VERBATIM -> VALIDATOR_VERBATIM_TO_INTERPRETED 292 List<Graph<StepType>.Edge> validatorTabularToVerbatimEdges = wf.getNodeEdges(VALIDATOR_TABULAR_TO_VERBATIM); 293 Assertions.assertEquals(1, validatorTabularToVerbatimEdges.size()); 294 Assertions.assertEquals(VALIDATOR_VERBATIM_TO_INTERPRETED, validatorTabularToVerbatimEdges.get(0).getNode()); 295 296 // VALIDATOR_VERBATIM_TO_INTERPRETED -> VALIDATOR_INTERPRETED_TO_INDEX 297 List<Graph<StepType>.Edge> validatorVerbatimToInterpretedEdges = wf.getNodeEdges(VALIDATOR_VERBATIM_TO_INTERPRETED); 298 Assertions.assertEquals(1, validatorVerbatimToInterpretedEdges.size()); 299 Assertions.assertEquals(VALIDATOR_INTERPRETED_TO_INDEX, validatorVerbatimToInterpretedEdges.get(0).getNode()); 300 301 // VALIDATOR_INTERPRETED_TO_INDEX -> 302 List<Graph<StepType>.Edge> validatorInterpretedToIndexEdges = wf.getNodeEdges(VALIDATOR_INTERPRETED_TO_INDEX); 303 Assertions.assertEquals(1, validatorInterpretedToIndexEdges.size()); 304 Assertions.assertEquals(VALIDATOR_COLLECT_METRICS, validatorInterpretedToIndexEdges.get(0).getNode()); 305 306 // VALIDATOR_COLLECT_METRICS -> 0 307 List<Graph<StepType>.Edge> validatorCollectMetricsEdges = wf.getNodeEdges(VALIDATOR_COLLECT_METRICS); 308 Assertions.assertEquals(0, validatorCollectMetricsEdges.size()); 309 } 310 311}