001/*
002 * Copyright 2014 Global Biodiversity Information Facility (GBIF)
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.gbif.api.model.crawler;
017
018import java.util.Date;
019import java.util.UUID;
020import javax.annotation.Nullable;
021import javax.validation.constraints.Min;
022
023import com.google.common.base.Objects;
024
025import static com.google.common.base.Preconditions.checkArgument;
026import static com.google.common.base.Preconditions.checkNotNull;
027
028
029/**
030 * Information about a dataset that is currently being processed. That usually means that we are crawling it at the
031 * moment or are in the process of persisting and interpreting its occurrences.
032 */
033public class DatasetProcessStatus {
034
035  private UUID datasetKey;
036  private CrawlJob crawlJob;
037  private Date startedCrawling;
038  private Date finishedCrawling;
039  private String crawlContext;
040  private FinishReason finishReason;
041  private ProcessState processStateOccurrence;
042  private ProcessState processStateChecklist;
043  private ProcessState processStateSample;
044
045  // Long instead of Optional<Long> because of JSON serialization issues, Jackson doesn't honor the NON_NULL setting
046  // for the Guava extension
047  private Long declaredCount;
048
049  private long pagesCrawled;
050
051  private long pagesFragmentedSuccessful;
052  private long pagesFragmentedError;
053  private long fragmentsEmitted;
054
055  private long fragmentsReceived;
056  private long rawOccurrencesPersistedNew;
057  private long rawOccurrencesPersistedUpdated;
058  private long rawOccurrencesPersistedUnchanged;
059  private long rawOccurrencesPersistedError;
060  private long fragmentsProcessed;
061
062  private long verbatimOccurrencesPersistedSuccessful;
063  private long verbatimOccurrencesPersistedError;
064
065  private long interpretedOccurrencesPersistedSuccessful;
066  private long interpretedOccurrencesPersistedError;
067
068  public static Builder builder() {
069    return new Builder();
070  }
071
072  public DatasetProcessStatus() {
073    // This constructor is needed for Jackson deserialization
074  }
075
076  /**
077   * Builds a new object from the builder validating it in the process.
078   * <p/>
079   * We only validate very little (all counts have to be greater than or equal to zero, a few null checks etc.) but not
080   */
081  public DatasetProcessStatus(Builder builder) {
082    checkNotNull(builder, "builder can't be null");
083
084    datasetKey = checkNotNull(builder.datasetKey, "datasetKey can't be null");
085    crawlJob = checkNotNull(builder.crawlJob, "crawlJob can't be null");
086    startedCrawling = builder.startedCrawling;
087    finishedCrawling = builder.finishedCrawling;
088    finishReason = builder.finishReason;
089    processStateOccurrence = builder.processStateOccurrence;
090    processStateChecklist = builder.processStateChecklist;
091    processStateSample = builder.processStateSample;
092    crawlContext = builder.crawlContext;
093
094    declaredCount = builder.declaredCount;
095
096    pagesCrawled = builder.pagesCrawled;
097
098    pagesFragmentedSuccessful = builder.pagesFragmentedSuccessful;
099    pagesFragmentedError = builder.pagesFragmentedError;
100    fragmentsEmitted = builder.fragmentsEmitted;
101
102    fragmentsReceived = builder.fragmentsReceived;
103    rawOccurrencesPersistedNew = builder.rawOccurrencesPersistedNew;
104    rawOccurrencesPersistedUpdated = builder.rawOccurrencesPersistedUpdated;
105    rawOccurrencesPersistedUnchanged = builder.rawOccurrencesPersistedUnchanged;
106    rawOccurrencesPersistedError = builder.rawOccurrencesPersistedError;
107    fragmentsProcessed = builder.fragmentsProcessed;
108
109    verbatimOccurrencesPersistedSuccessful = builder.verbatimOccurrencesPersistedSuccessful;
110    verbatimOccurrencesPersistedError = builder.verbatimOccurrencesPersistedError;
111
112    interpretedOccurrencesPersistedSuccessful = builder.interpretedOccurrencesPersistedSuccessful;
113    interpretedOccurrencesPersistedError = builder.interpretedOccurrencesPersistedError;
114
115    checkArgument(declaredCount == null || declaredCount >= 0,
116                  "declaredCount must be either null or greater than or equal to zero");
117
118    checkArgument(pagesCrawled >= 0, "pagesCrawled has to be greater than or equal to zero");
119
120    checkArgument(pagesFragmentedSuccessful >= 0, "pagesFragmentedSuccessful");
121    checkArgument(pagesFragmentedError >= 0, "pagesFragmentedError has to be greater than or equal to zero");
122    checkArgument(fragmentsEmitted >= 0, "fragmentsEmitted has to be greater than or equal to zero");
123
124    checkArgument(fragmentsReceived >= 0, "fragmentsReceived has to be greater than or equal to zero");
125    checkArgument(rawOccurrencesPersistedNew >= 0,
126                  "rawOccurrencesPersistedNew has to be greater than or equal to zero");
127    checkArgument(rawOccurrencesPersistedUpdated >= 0,
128                  "rawOccurrencesPersistedUpdated has to be greater than or equal to zero");
129    checkArgument(rawOccurrencesPersistedUnchanged >= 0,
130                  "rawOccurrencesPersistedUnchanged has to be greater than or equal to zero");
131    checkArgument(rawOccurrencesPersistedError >= 0,
132                  "rawOccurrencesPersistedError has to be greater than or equal to zero");
133    checkArgument(fragmentsProcessed >= 0, "fragmentsProcessed has to be greater than or equal to zero");
134
135    checkArgument(verbatimOccurrencesPersistedSuccessful >= 0,
136                  "verbatimOccurrencesPersistedSuccessful has to be greater than or equal to zero");
137    checkArgument(verbatimOccurrencesPersistedError >= 0,
138                  "verbatimOccurrencesPersistedError has to be greater than or equal to zero");
139
140    checkArgument(interpretedOccurrencesPersistedSuccessful >= 0,
141                  "interpretedOccurrencesPersistedSuccessful has to be greater than or equal to zero");
142    checkArgument(interpretedOccurrencesPersistedError >= 0,
143                  "interpretedOccurrencesPersistedError has to be greater than or equal to zero");
144  }
145
146  /**
147   * Last successful crawl context, this is a JSON string.
148   *
149   * @return the last successful crawl context, this is a JSON string
150   */
151  @Nullable
152  public String getCrawlContext() {
153    return crawlContext;
154  }
155
156  public CrawlJob getCrawlJob() {
157    return crawlJob;
158  }
159
160  /**
161   * Key that identifies the Dataset.
162   *
163   * @return the UUID key that identifies the dataset
164   */
165  public UUID getDatasetKey() {
166    return datasetKey;
167  }
168
169  @Nullable
170  public Long getDeclaredCount() {
171    return declaredCount;
172  }
173
174  /**
175   * Timestamp of when the crawl was finished.
176   *
177   * @return the timestamp when the crawl finished
178   */
179  @Nullable
180  public Date getFinishedCrawling() {
181    return finishedCrawling;
182  }
183
184  /**
185   * The reason a crawl is finished. Will be {@code null} if {@link #getFinishedCrawling()} returns {@code null}.
186   *
187   * @return the reason the crawl finished
188   */
189  @Nullable
190  public FinishReason getFinishReason() {
191    return finishReason;
192  }
193
194  @Nullable
195  public ProcessState getProcessStateOccurrence() {
196    return processStateOccurrence;
197  }
198
199  @Nullable
200  public ProcessState getProcessStateChecklist() {
201    return processStateChecklist;
202  }
203
204  @Nullable
205  public ProcessState getProcessStateSample() {
206    return processStateSample;
207  }
208
209  @Min(0)
210  public long getFragmentsEmitted() {
211    return fragmentsEmitted;
212  }
213
214  /**
215   * Number of fragments that have been processed.
216   *
217   * @return the number of fragments that have been processed
218   */
219  @Min(0)
220  public long getFragmentsProcessed() {
221    return fragmentsProcessed;
222  }
223
224  @Min(0)
225  public long getFragmentsReceived() {
226    return fragmentsReceived;
227  }
228
229  @Min(0)
230  public long getInterpretedOccurrencesPersistedError() {
231    return interpretedOccurrencesPersistedError;
232  }
233
234  @Min(0)
235  public long getInterpretedOccurrencesPersistedSuccessful() {
236    return interpretedOccurrencesPersistedSuccessful;
237  }
238
239  /**
240   * Number of pages crawled in total.
241   *
242   * @return number of pages crawled
243   */
244  @Min(0)
245  public long getPagesCrawled() {
246    return pagesCrawled;
247  }
248
249  @Min(0)
250  public long getPagesFragmentedError() {
251    return pagesFragmentedError;
252  }
253
254  /**
255   * Number of pages that have been fragmented.
256   *
257   * @return the number of pages that have been fragmented
258   */
259  @Min(0)
260  public long getPagesFragmentedSuccessful() {
261    return pagesFragmentedSuccessful;
262  }
263
264  @Min(0)
265  public long getRawOccurrencesPersistedError() {
266    return rawOccurrencesPersistedError;
267  }
268
269  @Min(0)
270  public long getRawOccurrencesPersistedNew() {
271    return rawOccurrencesPersistedNew;
272  }
273
274  @Min(0)
275  public long getRawOccurrencesPersistedUnchanged() {
276    return rawOccurrencesPersistedUnchanged;
277  }
278
279  @Min(0)
280  public long getRawOccurrencesPersistedUpdated() {
281    return rawOccurrencesPersistedUpdated;
282  }
283
284  /**
285   * Timestamp of when the crawl was actually started by a crawler.
286   *
287   * @return the timestamp when the crawl started
288   */
289  @Nullable
290  public Date getStartedCrawling() {
291    return startedCrawling;
292  }
293
294  @Min(0)
295  public long getVerbatimOccurrencesPersistedError() {
296    return verbatimOccurrencesPersistedError;
297  }
298
299  @Min(0)
300  public long getVerbatimOccurrencesPersistedSuccessful() {
301    return verbatimOccurrencesPersistedSuccessful;
302  }
303
304
305  public void setDatasetKey(UUID datasetKey) {
306    this.datasetKey = datasetKey;
307  }
308
309
310  public void setCrawlJob(CrawlJob crawlJob) {
311    this.crawlJob = crawlJob;
312  }
313
314
315  public void setStartedCrawling(Date startedCrawling) {
316    this.startedCrawling = startedCrawling;
317  }
318
319
320  public void setFinishedCrawling(Date finishedCrawling) {
321    this.finishedCrawling = finishedCrawling;
322  }
323
324
325  public void setCrawlContext(String crawlContext) {
326    this.crawlContext = crawlContext;
327  }
328
329
330  public void setFinishReason(FinishReason finishReason) {
331    this.finishReason = finishReason;
332  }
333
334  public void setProcessStateOccurrence(ProcessState processStateOccurrence) {
335    this.processStateOccurrence = processStateOccurrence;
336  }
337
338  public void setProcessStateChecklist(ProcessState processStateChecklist) {
339    this.processStateChecklist = processStateChecklist;
340  }
341
342  public void setProcessStateSample(ProcessState processStateSample) {
343    this.processStateSample = processStateSample;
344  }
345
346  public void setDeclaredCount(Long declaredCount) {
347    this.declaredCount = declaredCount;
348  }
349
350
351  public void setPagesCrawled(long pagesCrawled) {
352    this.pagesCrawled = pagesCrawled;
353  }
354
355
356  public void setPagesFragmentedSuccessful(long pagesFragmentedSuccessful) {
357    this.pagesFragmentedSuccessful = pagesFragmentedSuccessful;
358  }
359
360
361  public void setPagesFragmentedError(long pagesFragmentedError) {
362    this.pagesFragmentedError = pagesFragmentedError;
363  }
364
365
366  public void setFragmentsEmitted(long fragmentsEmitted) {
367    this.fragmentsEmitted = fragmentsEmitted;
368  }
369
370
371  public void setFragmentsReceived(long fragmentsReceived) {
372    this.fragmentsReceived = fragmentsReceived;
373  }
374
375
376  public void setRawOccurrencesPersistedNew(long rawOccurrencesPersistedNew) {
377    this.rawOccurrencesPersistedNew = rawOccurrencesPersistedNew;
378  }
379
380
381  public void setRawOccurrencesPersistedUpdated(long rawOccurrencesPersistedUpdated) {
382    this.rawOccurrencesPersistedUpdated = rawOccurrencesPersistedUpdated;
383  }
384
385
386  public void setRawOccurrencesPersistedUnchanged(long rawOccurrencesPersistedUnchanged) {
387    this.rawOccurrencesPersistedUnchanged = rawOccurrencesPersistedUnchanged;
388  }
389
390
391  public void setRawOccurrencesPersistedError(long rawOccurrencesPersistedError) {
392    this.rawOccurrencesPersistedError = rawOccurrencesPersistedError;
393  }
394
395
396  public void setFragmentsProcessed(long fragmentsProcessed) {
397    this.fragmentsProcessed = fragmentsProcessed;
398  }
399
400
401  public void setVerbatimOccurrencesPersistedSuccessful(long verbatimOccurrencesPersistedSuccessful) {
402    this.verbatimOccurrencesPersistedSuccessful = verbatimOccurrencesPersistedSuccessful;
403  }
404
405
406  public void setVerbatimOccurrencesPersistedError(long verbatimOccurrencesPersistedError) {
407    this.verbatimOccurrencesPersistedError = verbatimOccurrencesPersistedError;
408  }
409
410
411  public void setInterpretedOccurrencesPersistedSuccessful(long interpretedOccurrencesPersistedSuccessful) {
412    this.interpretedOccurrencesPersistedSuccessful = interpretedOccurrencesPersistedSuccessful;
413  }
414
415
416  public void setInterpretedOccurrencesPersistedError(long interpretedOccurrencesPersistedError) {
417    this.interpretedOccurrencesPersistedError = interpretedOccurrencesPersistedError;
418  }
419
420  @Override
421  public boolean equals(Object obj) {
422    if (obj == null) {
423      return false;
424    }
425    if (getClass() != obj.getClass()) {
426      return false;
427    }
428    final DatasetProcessStatus other = (DatasetProcessStatus) obj;
429    return Objects.equal(this.datasetKey, other.datasetKey)
430           && Objects.equal(this.crawlJob, other.crawlJob)
431           && Objects.equal(this.startedCrawling, other.startedCrawling)
432           && Objects.equal(this.finishedCrawling, other.finishedCrawling)
433           && Objects.equal(this.finishReason, other.finishReason)
434           && Objects.equal(this.processStateOccurrence, other.processStateOccurrence)
435           && Objects.equal(this.processStateChecklist, other.processStateChecklist)
436           && Objects.equal(this.processStateSample, other.processStateSample)
437           && Objects.equal(this.crawlContext, other.crawlContext)
438           && Objects.equal(this.declaredCount, other.declaredCount)
439           && Objects.equal(this.pagesCrawled, other.pagesCrawled)
440           && Objects.equal(this.pagesFragmentedSuccessful, other.pagesFragmentedSuccessful)
441           && Objects.equal(this.pagesFragmentedError, other.pagesFragmentedError)
442           && Objects.equal(this.fragmentsEmitted, other.fragmentsEmitted)
443           && Objects.equal(this.fragmentsReceived, other.fragmentsReceived)
444           && Objects.equal(this.rawOccurrencesPersistedNew, other.rawOccurrencesPersistedNew)
445           && Objects.equal(this.rawOccurrencesPersistedUpdated, other.rawOccurrencesPersistedUpdated)
446           && Objects.equal(this.rawOccurrencesPersistedUnchanged, other.rawOccurrencesPersistedUnchanged)
447           && Objects.equal(this.rawOccurrencesPersistedError, other.rawOccurrencesPersistedError)
448           && Objects.equal(this.fragmentsProcessed, other.fragmentsProcessed)
449           && Objects.equal(this.verbatimOccurrencesPersistedSuccessful, other.verbatimOccurrencesPersistedSuccessful)
450           && Objects.equal(this.verbatimOccurrencesPersistedError, other.verbatimOccurrencesPersistedError)
451           && Objects.equal(this.interpretedOccurrencesPersistedSuccessful,
452                            other.interpretedOccurrencesPersistedSuccessful)
453           && Objects.equal(this.interpretedOccurrencesPersistedError, other.interpretedOccurrencesPersistedError);
454  }
455
456  @Override
457  public int hashCode() {
458    return Objects.hashCode(datasetKey,
459                            crawlJob,
460                            startedCrawling,
461                            finishedCrawling,
462                            finishReason,
463                            processStateOccurrence,
464                            processStateChecklist,
465                            processStateSample,
466                            crawlContext,
467                            declaredCount,
468                            pagesCrawled,
469                            pagesFragmentedSuccessful,
470                            pagesFragmentedError,
471                            fragmentsEmitted,
472                            fragmentsReceived,
473                            rawOccurrencesPersistedNew,
474                            rawOccurrencesPersistedUpdated,
475                            rawOccurrencesPersistedUnchanged,
476                            rawOccurrencesPersistedError,
477                            fragmentsProcessed,
478                            verbatimOccurrencesPersistedSuccessful,
479                            verbatimOccurrencesPersistedError,
480                            interpretedOccurrencesPersistedSuccessful,
481                            interpretedOccurrencesPersistedError);
482  }
483
484  @Override
485  public String toString() {
486    return Objects.toStringHelper(this)
487      .add("datasetKey", datasetKey)
488      .add("crawlJob", crawlJob)
489      .add("startedCrawling", startedCrawling)
490      .add("finishedCrawling", finishedCrawling)
491      .add("finisReason", finishReason)
492      .add("processStateOccurrence", processStateOccurrence)
493      .add("processStateChecklist", processStateChecklist)
494      .add("processStateSample", processStateSample)
495      .add("crawlContext", crawlContext)
496      .add("declaredCount", declaredCount)
497      .add("pagesCrawled", pagesCrawled)
498      .add("pagesFragmentedSuccessful", pagesFragmentedSuccessful)
499      .add("pagesFragmentedError", pagesFragmentedError)
500      .add("fragmentsEmitted", fragmentsEmitted)
501      .add("fragmentsReceived", fragmentsReceived)
502      .add("rawOccurrencesPersistedNew", rawOccurrencesPersistedNew)
503      .add("rawOccurrencesPersistedUpdated", rawOccurrencesPersistedUpdated)
504      .add("rawOccurrencesPersistedUnchanged", rawOccurrencesPersistedUnchanged)
505      .add("rawOccurrencesPersistedError", rawOccurrencesPersistedError)
506      .add("fragmentsProcessed", fragmentsProcessed)
507      .add("verbatimOccurrencesPersistedSuccessful", verbatimOccurrencesPersistedSuccessful)
508      .add("verbatimOccurrencesPersistedError", verbatimOccurrencesPersistedError)
509      .add("interpretedOccurrencesPersistedSuccessful", interpretedOccurrencesPersistedSuccessful)
510      .add("interpretedOccurrencesPersistedError", interpretedOccurrencesPersistedError)
511      .toString();
512  }
513
514  public static class Builder {
515
516    private UUID datasetKey;
517    private CrawlJob crawlJob;
518    private Date startedCrawling;
519    private Date finishedCrawling;
520    private String crawlContext;
521    private FinishReason finishReason;
522    private ProcessState processStateOccurrence;
523    private ProcessState processStateChecklist;
524    private ProcessState processStateSample;
525    private Long declaredCount;
526    private long pagesCrawled;
527    private long pagesFragmentedSuccessful;
528    private long pagesFragmentedError;
529    private long fragmentsEmitted;
530    private long fragmentsReceived;
531    private long rawOccurrencesPersistedNew;
532    private long rawOccurrencesPersistedUpdated;
533    private long rawOccurrencesPersistedUnchanged;
534    private long rawOccurrencesPersistedError;
535    private long fragmentsProcessed;
536    private long verbatimOccurrencesPersistedSuccessful;
537    private long verbatimOccurrencesPersistedError;
538    private long interpretedOccurrencesPersistedSuccessful;
539    private long interpretedOccurrencesPersistedError;
540
541    public DatasetProcessStatus build() {
542      return new DatasetProcessStatus(this);
543    }
544
545    public Builder crawlContext(String crawlContext) {
546      this.crawlContext = crawlContext;
547      return this;
548    }
549
550    public Builder crawlJob(CrawlJob crawlJob) {
551      this.crawlJob = crawlJob;
552      return this;
553    }
554
555    public Builder datasetKey(UUID datasetKey) {
556      this.datasetKey = datasetKey;
557      return this;
558    }
559
560    public Builder declaredCount(Long declaredCount) {
561      this.declaredCount = declaredCount;
562      return this;
563    }
564
565    public Builder finishedCrawling(Date finishedCrawling) {
566      this.finishedCrawling = finishedCrawling;
567      return this;
568    }
569
570    public Builder finishReason(FinishReason finishReason) {
571      this.finishReason = finishReason;
572      return this;
573    }
574
575    public Builder processStateOccurrence(ProcessState processStateOccurrence) {
576      this.processStateOccurrence = processStateOccurrence;
577      return this;
578    }
579
580    public Builder processStateChecklist(ProcessState processStateChecklist) {
581      this.processStateChecklist = processStateChecklist;
582      return this;
583    }
584
585    public Builder processStateSample(ProcessState processStateSample) {
586      this.processStateSample = processStateSample;
587      return this;
588    }
589
590    public Builder fragmentsEmitted(long fragmentsEmitted) {
591      this.fragmentsEmitted = fragmentsEmitted;
592      return this;
593    }
594
595    public Builder fragmentsProcessed(long fragmentsProcessed) {
596      this.fragmentsProcessed = fragmentsProcessed;
597      return this;
598    }
599
600    public Builder fragmentsReceived(long fragmentsReceived) {
601      this.fragmentsReceived = fragmentsReceived;
602      return this;
603    }
604
605    public Builder interpretedOccurrencesPersistedError(long interpretedOccurrencesPersistedError) {
606      this.interpretedOccurrencesPersistedError = interpretedOccurrencesPersistedError;
607      return this;
608    }
609
610    public Builder interpretedOccurrencesPersistedSuccessful(long interpretedOccurrencesPersistedSuccessful) {
611      this.interpretedOccurrencesPersistedSuccessful = interpretedOccurrencesPersistedSuccessful;
612      return this;
613    }
614
615    public Builder pagesCrawled(long pagesCrawled) {
616      this.pagesCrawled = pagesCrawled;
617      return this;
618    }
619
620    public Builder pagesFragmentedError(long pagesFragmentedError) {
621      this.pagesFragmentedError = pagesFragmentedError;
622      return this;
623    }
624
625    public Builder pagesFragmentedSuccessful(long pagesFragmentedSuccessful) {
626      this.pagesFragmentedSuccessful = pagesFragmentedSuccessful;
627      return this;
628    }
629
630    public Builder rawOccurrencesPersistedError(long rawOccurrencesPersistedError) {
631      this.rawOccurrencesPersistedError = rawOccurrencesPersistedError;
632      return this;
633    }
634
635    public Builder rawOccurrencesPersistedNew(long rawOccurrencesPersistedNew) {
636      this.rawOccurrencesPersistedNew = rawOccurrencesPersistedNew;
637      return this;
638    }
639
640    public Builder rawOccurrencesPersistedUnchanged(long rawOccurrencesPersistedUnchanged) {
641      this.rawOccurrencesPersistedUnchanged = rawOccurrencesPersistedUnchanged;
642      return this;
643    }
644
645    public Builder rawOccurrencesPersistedUpdated(long rawOccurrencesPersistedUpdated) {
646      this.rawOccurrencesPersistedUpdated = rawOccurrencesPersistedUpdated;
647      return this;
648    }
649
650    public Builder startedCrawling(Date startedCrawling) {
651      this.startedCrawling = startedCrawling;
652      return this;
653    }
654
655    public Builder verbatimOccurrencesPersistedError(long verbatimOccurrencesPersistedError) {
656      this.verbatimOccurrencesPersistedError = verbatimOccurrencesPersistedError;
657      return this;
658    }
659
660    public Builder verbatimOccurrencesPersistedSuccessful(long verbatimOccurrencesPersistedSuccessful) {
661      this.verbatimOccurrencesPersistedSuccessful = verbatimOccurrencesPersistedSuccessful;
662      return this;
663    }
664  }
665}