001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 *     http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014package org.gbif.api.model.pipelines;
015
016import org.gbif.api.model.pipelines.PipelinesWorkflow.Graph;
017
018import java.util.Arrays;
019import java.util.HashSet;
020import java.util.List;
021import java.util.Set;
022import java.util.function.BiPredicate;
023import java.util.stream.Collectors;
024
025import org.junit.jupiter.api.Assertions;
026import org.junit.jupiter.api.Test;
027
028import static org.gbif.api.model.pipelines.StepType.*;
029
030
031public class PipelinesWorkflowTest {
032
033  private final BiPredicate<List<Graph<StepType>.Edge>, StepType> containsFn =
034    (l, t) -> l.stream().anyMatch(x -> x.getNode().equals(t));
035
036  @Test
037  public void findAllStepsToRunTest() {
038    Set<StepType> types = new HashSet<>(
039      Arrays.asList(VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW, INTERPRETED_TO_INDEX));
040
041    Set<StepType> allNodesFor = PipelinesWorkflow.getOccurrenceWorkflow().getAllNodesFor(types);
042
043    Assertions.assertEquals(5, allNodesFor.size());
044    Assertions.assertTrue(allNodesFor.containsAll(
045      Arrays.asList(VERBATIM_TO_IDENTIFIER, VERBATIM_TO_INTERPRETED, HDFS_VIEW, INTERPRETED_TO_INDEX, FRAGMENTER)));
046  }
047
048  @Test
049  public void findAllStepsToRunSameLevelTest() {
050    Set<StepType> types =
051      new HashSet<>(Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED));
052
053    Set<StepType> allNodesFor = PipelinesWorkflow.getEventOccurrenceWorkflow().getAllNodesFor(types);
054
055    Assertions.assertEquals(5, allNodesFor.size());
056    Assertions.assertTrue(allNodesFor.containsAll(
057      Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_HDFS_VIEW,
058        EVENTS_INTERPRETED_TO_INDEX)));
059  }
060
061  @Test
062  public void findOneRootStepsToRunTest() {
063    Set<StepType> types = new HashSet<>(
064      Arrays.asList(DWCA_TO_VERBATIM, VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW,
065        INTERPRETED_TO_INDEX));
066
067    Set<StepType> rootNodes = PipelinesWorkflow.getOccurrenceWorkflow().getRootNodesFor(types);
068
069    Assertions.assertEquals(1, rootNodes.size());
070    Assertions.assertTrue(rootNodes.contains(DWCA_TO_VERBATIM));
071  }
072
073  @Test
074  public void findMainRootStepsToRunTest() {
075    Set<StepType> types = new HashSet<>(
076      Arrays.asList(VERBATIM_TO_INTERPRETED, VERBATIM_TO_IDENTIFIER, HDFS_VIEW, INTERPRETED_TO_INDEX));
077
078    Set<StepType> rootNodes = PipelinesWorkflow.getOccurrenceWorkflow().getRootNodesFor(types);
079
080    Assertions.assertEquals(1, rootNodes.size());
081    Assertions.assertTrue(rootNodes.contains(VERBATIM_TO_IDENTIFIER));
082  }
083
084  @Test
085  public void findRootStepsToRunSameLevelTest() {
086    Set<StepType> types =
087      new HashSet<>(Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_HDFS_VIEW,
088        EVENTS_INTERPRETED_TO_INDEX));
089
090    Set<StepType> rootNodes = PipelinesWorkflow.getEventOccurrenceWorkflow().getRootNodesFor(types);
091
092    Assertions.assertEquals(3, rootNodes.size());
093    Assertions.assertTrue(rootNodes.containsAll(
094      Arrays.asList(HDFS_VIEW, INTERPRETED_TO_INDEX, EVENTS_VERBATIM_TO_INTERPRETED)));
095  }
096
097  @Test
098  public void eventOccurrenceWorkflowTest() {
099    Graph<StepType> wf = PipelinesWorkflow.getEventOccurrenceWorkflow();
100
101    Assertions.assertEquals(12, wf.getNodesQuantity());
102
103    Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM));
104    Assertions.assertEquals(2, wf.getLevel(DWCDP_TO_VERBATIM));
105    Assertions.assertEquals(2, wf.getLevel(VERBATIM_TO_IDENTIFIER));
106    Assertions.assertEquals(3, wf.getLevel(VERBATIM_TO_INTERPRETED));
107    Assertions.assertEquals(4, wf.getLevel(INTERPRETED_TO_INDEX));
108    Assertions.assertEquals(4, wf.getLevel(HDFS_VIEW));
109    Assertions.assertEquals(4, wf.getLevel(FRAGMENTER));
110    Assertions.assertEquals(4, wf.getLevel(EVENTS_VERBATIM_TO_INTERPRETED));
111    Assertions.assertEquals(5, wf.getLevel(EVENTS_INTERPRETED_TO_INDEX));
112    Assertions.assertEquals(5, wf.getLevel(EVENTS_HDFS_VIEW));
113
114    assertConnection(wf, DWCDP_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
115    assertConnection(wf, DWCA_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
116    assertConnection(wf, VERBATIM_TO_IDENTIFIER, VERBATIM_TO_INTERPRETED);
117    assertConnection(wf, VERBATIM_TO_INTERPRETED, INTERPRETED_TO_INDEX, HDFS_VIEW, FRAGMENTER, EVENTS_VERBATIM_TO_INTERPRETED);
118    assertConnection(wf, INTERPRETED_TO_INDEX);
119    assertConnection(wf, HDFS_VIEW);
120    assertConnection(wf, FRAGMENTER);
121    assertConnection(wf, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_INTERPRETED_TO_INDEX, EVENTS_HDFS_VIEW);
122    assertConnection(wf, EVENTS_INTERPRETED_TO_INDEX);
123    assertConnection(wf, EVENTS_HDFS_VIEW);
124  }
125
126  @Test
127  public void eventOnlyWorkflowTest() {
128    Graph<StepType> wf = PipelinesWorkflow.getEventWorkflow();
129
130    Assertions.assertEquals(7, wf.getNodesQuantity());
131
132    Assertions.assertEquals(1, wf.getLevel(DWCDP_STAGE));
133    Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM));
134    Assertions.assertEquals(2, wf.getLevel(DWCDP_TO_VERBATIM));
135    Assertions.assertEquals(2, wf.getLevel(EVENTS_VERBATIM_TO_INTERPRETED));
136    Assertions.assertEquals(3, wf.getLevel(EVENTS_INTERPRETED_TO_INDEX));
137    Assertions.assertEquals(3, wf.getLevel(EVENTS_HDFS_VIEW));
138
139    assertConnection(wf, DWCDP_STAGE, DWCDP_TO_VERBATIM);
140    assertConnection(wf, DWCA_TO_VERBATIM, EVENTS_VERBATIM_TO_INTERPRETED);
141    assertConnection(wf, DWCDP_TO_VERBATIM, EVENTS_VERBATIM_TO_INTERPRETED);
142    assertConnection(wf, EVENTS_VERBATIM_TO_INTERPRETED, EVENTS_INTERPRETED_TO_INDEX, EVENTS_HDFS_VIEW);
143    assertConnection(wf, EVENTS_INTERPRETED_TO_INDEX);
144    assertConnection(wf, EVENTS_HDFS_VIEW);
145  }
146
147  @Test
148  public void occurrenceWorkflowTest() {
149    Graph<StepType> wf = PipelinesWorkflow.getOccurrenceWorkflow();
150
151    Assertions.assertEquals(11, wf.getNodesQuantity());
152
153    Assertions.assertEquals(1, wf.getLevel(DWCA_TO_VERBATIM));
154    Assertions.assertEquals(1, wf.getLevel(ABCD_TO_VERBATIM));
155    Assertions.assertEquals(1, wf.getLevel(XML_TO_VERBATIM));
156    Assertions.assertEquals(2, wf.getLevel(VERBATIM_TO_IDENTIFIER));
157    Assertions.assertEquals(3, wf.getLevel(VERBATIM_TO_INTERPRETED));
158    Assertions.assertEquals(4, wf.getLevel(INTERPRETED_TO_INDEX));
159    Assertions.assertEquals(4, wf.getLevel(HDFS_VIEW));
160    Assertions.assertEquals(4, wf.getLevel(FRAGMENTER));
161
162    assertConnection(wf, DWCDP_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
163    assertConnection(wf, DWCA_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
164    assertConnection(wf, ABCD_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
165    assertConnection(wf, XML_TO_VERBATIM, VERBATIM_TO_IDENTIFIER);
166    assertConnection(wf, VERBATIM_TO_IDENTIFIER, VERBATIM_TO_INTERPRETED);
167    assertConnection(wf, VERBATIM_TO_INTERPRETED, INTERPRETED_TO_INDEX, HDFS_VIEW, FRAGMENTER);
168    assertConnection(wf, INTERPRETED_TO_INDEX);
169    assertConnection(wf, HDFS_VIEW);
170    assertConnection(wf, FRAGMENTER);
171  }
172
173  @Test
174  public void validatorWorkflowTest() {
175    Graph<StepType> wf = PipelinesWorkflow.getValidatorWorkflow();
176
177    Assertions.assertEquals(9, wf.getNodesQuantity());
178
179    Assertions.assertEquals(1, wf.getLevel(VALIDATOR_UPLOAD_ARCHIVE));
180    Assertions.assertEquals(2, wf.getLevel(VALIDATOR_VALIDATE_ARCHIVE));
181    Assertions.assertEquals(3, wf.getLevel(VALIDATOR_DWCA_TO_VERBATIM));
182    Assertions.assertEquals(3, wf.getLevel(VALIDATOR_ABCD_TO_VERBATIM));
183    Assertions.assertEquals(3, wf.getLevel(VALIDATOR_XML_TO_VERBATIM));
184    Assertions.assertEquals(3, wf.getLevel(VALIDATOR_TABULAR_TO_VERBATIM));
185    Assertions.assertEquals(4, wf.getLevel(VALIDATOR_VERBATIM_TO_IDENTIFIER));
186    Assertions.assertEquals(5, wf.getLevel(VALIDATOR_VERBATIM_TO_INTERPRETED));
187    Assertions.assertEquals(6, wf.getLevel(VALIDATOR_COLLECT_METRICS));
188
189    assertConnection(wf, VALIDATOR_UPLOAD_ARCHIVE, VALIDATOR_VALIDATE_ARCHIVE);
190    assertConnection(wf, VALIDATOR_VALIDATE_ARCHIVE,
191      VALIDATOR_DWCA_TO_VERBATIM, VALIDATOR_ABCD_TO_VERBATIM,
192      VALIDATOR_XML_TO_VERBATIM, VALIDATOR_TABULAR_TO_VERBATIM);
193    assertConnection(wf, VALIDATOR_DWCA_TO_VERBATIM, VALIDATOR_VERBATIM_TO_IDENTIFIER);
194    assertConnection(wf, VALIDATOR_ABCD_TO_VERBATIM, VALIDATOR_VERBATIM_TO_IDENTIFIER);
195    assertConnection(wf, VALIDATOR_XML_TO_VERBATIM, VALIDATOR_VERBATIM_TO_IDENTIFIER);
196    assertConnection(wf, VALIDATOR_TABULAR_TO_VERBATIM, VALIDATOR_VERBATIM_TO_IDENTIFIER);
197    assertConnection(wf, VALIDATOR_VERBATIM_TO_IDENTIFIER, VALIDATOR_VERBATIM_TO_INTERPRETED);
198    assertConnection(wf, VALIDATOR_VERBATIM_TO_INTERPRETED, VALIDATOR_COLLECT_METRICS);
199    assertConnection(wf, VALIDATOR_COLLECT_METRICS);
200  }
201
202  private void assertConnection(Graph<StepType> wf, StepType sourceNode, StepType... expectedTargetNodes) {
203    List<Graph<StepType>.Edge> edges = wf.getNodeEdges(sourceNode);
204    Assertions.assertEquals(expectedTargetNodes.length, edges.size());
205    Set<StepType> expectedNodes = new HashSet<>(Arrays.asList(expectedTargetNodes));
206    Set<StepType> actualNodes = edges.stream().map(Graph.Edge::getNode).collect(Collectors.toSet());
207    Assertions.assertEquals(expectedNodes, actualNodes);
208  }
209
210}