001/* 002 * Licensed under the Apache License, Version 2.0 (the "License"); 003 * you may not use this file except in compliance with the License. 004 * You may obtain a copy of the License at 005 * 006 * http://www.apache.org/licenses/LICENSE-2.0 007 * 008 * Unless required by applicable law or agreed to in writing, software 009 * distributed under the License is distributed on an "AS IS" BASIS, 010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 011 * See the License for the specific language governing permissions and 012 * limitations under the License. 013 */ 014package org.gbif.api.service.pipelines; 015 016import org.gbif.api.model.common.paging.Pageable; 017import org.gbif.api.model.common.paging.PagingResponse; 018import org.gbif.api.model.pipelines.PipelineExecution; 019import org.gbif.api.model.pipelines.PipelineProcess; 020import org.gbif.api.model.pipelines.PipelineStep; 021import org.gbif.api.model.pipelines.RunPipelineResponse; 022import org.gbif.api.model.pipelines.StepRunner; 023import org.gbif.api.model.pipelines.StepType; 024import org.gbif.api.model.pipelines.ws.PipelineProcessParameters; 025import org.gbif.api.model.pipelines.ws.RunAllParams; 026 027import java.util.List; 028import java.util.Set; 029import java.util.UUID; 030 031import javax.annotation.Nullable; 032import javax.validation.constraints.NotBlank; 033import javax.validation.constraints.NotNull; 034import javax.validation.constraints.Null; 035 036@SuppressWarnings("unused") 037public interface PipelinesHistoryService { 038 039 String STEPS_REQUIRED_MESSAGE = "Steps parameter is required"; 040 String REASON_REQUIRED_MESSAGE = "Reason parameter is required"; 041 042 /** 043 * Lists the history of all {@link PipelineProcess}, sorted descending from the most recent one. 044 * 045 * @param pageable paging request 046 * @return a paged response that contains a list of {@link PipelineProcess}. 047 */ 048 PagingResponse<PipelineProcess> history(Pageable pageable); 049 050 /** 051 * Lists the history of all {@link PipelineProcess} of a dataset, sorted descending from the most 052 * recent one. 053 * 054 * @param datasetKey dataset identifier 055 * @param pageable paging request 056 * @return a paged response that contains a list of {@link PipelineProcess}. 057 */ 058 PagingResponse<PipelineProcess> history(@NotNull UUID datasetKey, Pageable pageable); 059 060 /** 061 * Gets the PipelineProcess identified by the dataset and attempt identifiers. 062 * 063 * @param datasetKey dataset identifier 064 * @param attempt crawl attempt identifier 065 * @return an instance of pipelines process if exists. 066 */ 067 PipelineProcess getPipelineProcess(@NotNull UUID datasetKey, int attempt); 068 069 /** Returns information about all running pipelines executions */ 070 PagingResponse<PipelineProcess> getRunningPipelineProcess( 071 @Nullable StepType stepType, @Nullable StepRunner stepRunner, Pageable pageable); 072 073 /** 074 * Creates/persists a pipelines process of dataset for an attempt identifier. If the process 075 * already exists it returns the existing one. 076 * 077 * @param params pipeline process parameters, contain dataset key and attempt 078 * @return the key of the {@link PipelineProcess} created. 079 */ 080 long createPipelineProcess(@NotNull PipelineProcessParameters params); 081 082 /** 083 * Adds/persists the information of a pipeline execution. 084 * 085 * @param processKey sequential identifier of a pipeline process 086 * @param pipelineExecution pipeline execution data 087 * @return the key of the PipelineExecution created. 088 */ 089 long addPipelineExecution(long processKey, @NotNull PipelineExecution pipelineExecution); 090 091 /** 092 * Gets execution key for running dataset 093 * 094 * @param datasetKey dataset identifier 095 * @return running execution key 096 */ 097 Long getRunningExecutionKey(@NotNull UUID datasetKey); 098 099 /** 100 * Update the information of a pipeline step. 101 * 102 * @param pipelineStep step to be added 103 * @return the key of the PipelineStep created. 104 */ 105 long updatePipelineStep(@NotNull PipelineStep pipelineStep); 106 107 /** 108 * Gets the PipelineStep of the specified keys. 109 * 110 * @param stepKey key of the pipeline step 111 * @return {@link PipelineStep}. 112 */ 113 PipelineStep getPipelineStep(long stepKey); 114 115 /** 116 * Gets the PipelineSteps list of the execution key. 117 * 118 * @param executionKey key of the pipeline execution 119 * @return {@link List<PipelineStep>}. 120 */ 121 List<PipelineStep> getPipelineStepsByExecutionKey(long executionKey); 122 123 /** Mark all pipeline executions as finished to clean running UI */ 124 void markAllPipelineExecutionAsFinished(); 125 126 /** 127 * Mark pipeline execution as finished when all pipelin steps are finished 128 * 129 * @param executionKey key of the pipeline execution 130 */ 131 void markPipelineExecutionIfFinished(long executionKey); 132 133 /** 134 * Change status to ABORTED and set finished date if state is RUNNING, QUEUED or SUBMITTED, and 135 * set pipeline execution as finished 136 * 137 * @param executionKey key of the pipeline execution 138 */ 139 void markPipelineStatusAsAborted(long executionKey); 140 141 /** 142 * Runs the last attempt for all datasets. 143 * 144 * @param steps steps to run 145 * @param reason reason to run 146 * @param useLastSuccessful true if we want to run the latest successful attempt 147 * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during 148 * processing a dataset 149 * @param runAllParams parameters, contain datasets to exclude 150 * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and 151 * etc 152 * @return {@link RunPipelineResponse}. 153 */ 154 RunPipelineResponse runAll( 155 @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps, 156 @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason, 157 boolean useLastSuccessful, 158 boolean markPreviousAttemptAsFailed, 159 @Nullable RunAllParams runAllParams, 160 @Nullable Set<String> interpretTypes); 161 162 /** 163 * Restart last failed pipelines step for a dataset. 164 * 165 * @param datasetKey dataset key 166 * @param steps steps to run 167 * @param reason reason to run 168 * @param useLastSuccessful true if we want to run the latest successful attempt 169 * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during 170 * processing a dataset 171 * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and 172 * etc 173 * @return {@link RunPipelineResponse}. 174 */ 175 RunPipelineResponse runPipelineAttempt( 176 @NotNull UUID datasetKey, 177 @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps, 178 @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason, 179 boolean useLastSuccessful, 180 boolean markPreviousAttemptAsFailed, 181 @Nullable Set<String> interpretTypes); 182 183 /** 184 * Re-run a pipeline step. 185 * 186 * @param datasetKey dataset key 187 * @param attempt attempt to run 188 * @param steps steps to run 189 * @param reason reason to run 190 * @param markPreviousAttemptAsFailed previous status can't be wrong, when CLI restarted during 191 * processing a dataset 192 * @param interpretTypes is used for partial interpretation such as only TAXONOMY, METADATA and 193 * etc 194 * @return {@link RunPipelineResponse}. 195 */ 196 RunPipelineResponse runPipelineAttempt( 197 @NotNull UUID datasetKey, 198 int attempt, 199 @NotBlank(message = STEPS_REQUIRED_MESSAGE) String steps, 200 @NotBlank(message = REASON_REQUIRED_MESSAGE) String reason, 201 boolean markPreviousAttemptAsFailed, 202 @Nullable Set<String> interpretTypes); 203 204 /** 205 * Sends email to data administrator about absent identifiers issue with a dataset 206 * 207 * <p>Deprecated: use {@link #notifyAbsentIdentifiers(UUID, int, String)} instead. 208 * 209 * @param datasetKey dataset key 210 * @param attempt attempt to run 211 * @param message with failed metrics and other info* 212 */ 213 @Deprecated 214 void sendAbsentIndentifiersEmail(@NotNull UUID datasetKey, int attempt, @NotNull String message); 215 216 /** 217 * Mark failed identifier stage as finished and continue interpretation process for datasets were 218 * identifier stage failed because of a threshold limit 219 * 220 * @param datasetKey dataset key 221 * @param attempt attempt to run 222 */ 223 void allowAbsentIndentifiers(@NotNull UUID datasetKey, int attempt); 224 225 /** 226 * Mark latest failed identifier stage as finished and continue interpretation process for 227 * datasets were identifier stage failed because of a threshold limit 228 * 229 * @param datasetKey dataset key 230 */ 231 void allowAbsentIndentifiers(@NotNull UUID datasetKey); 232 233 /** 234 * Sends a notification to the data administrators about absent identifiers issues with the 235 * dataset. 236 * 237 * @param datasetKey key of the dataset 238 * @param attempt crawling attempt 239 * @param executionKey key of the pipelines execution 240 * @param message cause of the issue 241 */ 242 void notifyAbsentIdentifiers(UUID datasetKey, int attempt, long executionKey, String message); 243 244 /** 245 * Sets the pipeline step state QUEUED only if it's in SUBMITTED state. 246 * 247 * @param key pipeline step key 248 */ 249 void setSubmittedPipelineStepToQueued(long key); 250}