001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 *     http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014package org.gbif.api.service.pipelines;
015
016import org.gbif.api.model.common.paging.Pageable;
017import org.gbif.api.model.common.paging.PagingResponse;
018import org.gbif.api.model.pipelines.PipelineExecution;
019import org.gbif.api.model.pipelines.PipelineProcess;
020import org.gbif.api.model.pipelines.PipelineStep;
021import org.gbif.api.model.pipelines.RunPipelineResponse;
022import org.gbif.api.model.pipelines.ws.PipelineProcessParameters;
023import org.gbif.api.model.pipelines.ws.RunAllParams;
024
025import java.util.List;
026import java.util.Set;
027import java.util.UUID;
028
029import javax.annotation.Nullable;
030import javax.validation.constraints.NotBlank;
031import javax.validation.constraints.NotNull;
032
033@SuppressWarnings("unused")
034public interface PipelinesHistoryService {
035
036  String STEPS_REQUIRED_MESSAGE = "Steps parameter is required";
037  String REASON_REQUIRED_MESSAGE = "Reason parameter is required";
038
039  /**
040   * Lists the history of all {@link PipelineProcess}, sorted descending from the most recent one.
041   *
042   * @param pageable paging request
043   * @return a paged response that contains a list of {@link PipelineProcess}.
044   */
045  PagingResponse<PipelineProcess> history(Pageable pageable);
046
047  /**
048   * Lists the history of all {@link PipelineProcess} of a dataset, sorted descending from the most
049   * recent one.
050   *
051   * @param datasetKey dataset identifier
052   * @param pageable paging request
053   * @return a paged response that contains a list of {@link PipelineProcess}.
054   */
055  PagingResponse<PipelineProcess> history(@NotNull UUID datasetKey, Pageable pageable);
056
057  /**
058   * Gets the PipelineProcess identified by the dataset and attempt identifiers.
059   *
060   * @param datasetKey dataset identifier
061   * @param attempt crawl attempt identifier
062   * @return an instance of pipelines process if exists.
063   */
064  PipelineProcess getPipelineProcess(@NotNull UUID datasetKey, int attempt);
065
066  /** Returns information about all running pipelines executions */
067  PagingResponse<PipelineProcess> getRunningPipelineProcess(Pageable pageable);
068
069  /**
070   * Creates/persists a pipelines process of dataset for an attempt identifier. If the process
071   * already exists it returns the existing one.
072   *
073   * @param params pipeline process parameters, contain dataset key and attempt
074   * @return the key of the {@link PipelineProcess} created.
075   */
076  long createPipelineProcess(@NotNull PipelineProcessParameters params);
077
078  /**
079   * Adds/persists the information of a pipeline execution.
080   *
081   * @param processKey sequential identifier of a pipeline process
082   * @param pipelineExecution pipeline execution data
083   * @return the key of the PipelineExecution created.
084   */
085  long addPipelineExecution(long processKey, @NotNull PipelineExecution pipelineExecution);
086
087  /**
088   * Gets execution key for running dataset
089   *
090   * @param datasetKey dataset identifier
091   * @return running execution key
092   */
093  Long getRunningExecutionKey(@NotNull UUID datasetKey);
094
095  /**
096   * Update the information of a pipeline step.
097   *
098   * @param pipelineStep step to be added
099   * @return the key of the PipelineStep created.
100   */
101  long updatePipelineStep(@NotNull PipelineStep pipelineStep);
102
103  /**
104   * Gets the PipelineStep of the specified keys.
105   *
106   * @param stepKey key of the pipeline step
107   * @return {@link PipelineStep}.
108   */
109  PipelineStep getPipelineStep(long stepKey);
110
111  /**
112   * Gets the PipelineSteps list of the execution key.
113   *
114   * @param executionKey key of the pipeline execution
115   * @return {@link List<PipelineStep>}.
116   */
117  List<PipelineStep> getPipelineStepsByExecutionKey(long executionKey);
118
119  /** Mark all pipeline executions as finished to clean running UI */
120  void markAllPipelineExecutionAsFinished();
121
122  /**
123   * Mark pipeline execution as finished when all pipelin steps are finished
124   *
125   * @param executionKey key of the pipeline execution
126   */
127  void markPipelineExecutionIfFinished(long executionKey);
128
129  /**
130   * Change status to ABORTED and set finished date if state is RUNNING, QUEUED or SUBMITTED, and
131   * set pipeline execution as finished
132   *
133   * @param executionKey key of the pipeline execution
134   */
135  void markPipelineStatusAsAborted(long executionKey);
136
137  /**
138   * Runs the last attempt for all datasets.
139   *
140   * @param steps steps to run
141   * @param reason reason to run
142   * @param useLastSuccessful true if we want to run the latest successful attempt
143   * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during
144   *     processing a dataset
145   * @param runAllParams parameters, contain datasets to exclude
146   * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and
147   *     etc
148   * @return {@link RunPipelineResponse}.
149   */
150  RunPipelineResponse runAll(
151      @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps,
152      @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason,
153      boolean useLastSuccessful,
154      boolean markPreviousAttemptAsFailed,
155      @Nullable RunAllParams runAllParams,
156      @Nullable Set<String> interpretTypes);
157
158  /**
159   * Restart last failed pipelines step for a dataset.
160   *
161   * @param datasetKey dataset key
162   * @param steps steps to run
163   * @param reason reason to run
164   * @param useLastSuccessful true if we want to run the latest successful attempt
165   * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during
166   *     processing a dataset
167   * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and
168   *     etc
169   * @return {@link RunPipelineResponse}.
170   */
171  RunPipelineResponse runPipelineAttempt(
172      @NotNull UUID datasetKey,
173      @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps,
174      @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason,
175      boolean useLastSuccessful,
176      boolean markPreviousAttemptAsFailed,
177      @Nullable Set<String> interpretTypes);
178
179  /**
180   * Re-run a pipeline step.
181   *
182   * @param datasetKey dataset key
183   * @param attempt attempt to run
184   * @param steps steps to run
185   * @param reason reason to run
186   * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during
187   *     processing a dataset
188   * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and
189   *     etc
190   * @return {@link RunPipelineResponse}.
191   */
192  RunPipelineResponse runPipelineAttempt(
193      @NotNull UUID datasetKey,
194      int attempt,
195      @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps,
196      @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason,
197      boolean markPreviousAttemptAsFailed,
198      @Nullable Set<String> interpretTypes);
199
200  /**
201   * Sends email to data administrator about absent identifiers issue with a dataset
202   *
203   * <p>Deprecated: use {@link #notifyAbsentIdentifiers(UUID, int, String)} instead.
204   *
205   * @param datasetKey dataset key
206   * @param attempt attempt to run
207   * @param message with failed metrics and other info*
208   */
209  @Deprecated
210  void sendAbsentIndentifiersEmail(@NotNull UUID datasetKey, int attempt, @NotNull String message);
211
212  /**
213   * Mark failed identifier stage as finished and continue interpretation process for datasets were
214   * identifier stage failed because of a threshold limit
215   *
216   * @param datasetKey dataset key
217   * @param attempt attempt to run
218   */
219  void allowAbsentIndentifiers(@NotNull UUID datasetKey, int attempt);
220
221  /**
222   * Mark latest failed identifier stage as finished and continue interpretation process for
223   * datasets were identifier stage failed because of a threshold limit
224   *
225   * @param datasetKey dataset key
226   */
227  void allowAbsentIndentifiers(@NotNull UUID datasetKey);
228
229  /**
230   * Sends a notification to the data administrators about absent identifiers issues with the
231   * dataset.
232   *
233   * @param datasetKey key of the dataset
234   * @param attempt crawling attempt
235   * @param executionKey key of the pipelines execution
236   * @param message cause of the issue
237   */
238  void notifyAbsentIdentifiers(UUID datasetKey, int attempt, long executionKey, String message);
239}