001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 *     http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014package org.gbif.api.model.occurrence;
015
016import org.gbif.api.model.Constants;
017import org.gbif.api.model.occurrence.search.OccurrenceSearchParameter;
018import org.gbif.api.model.predicate.EqualsPredicate;
019import org.gbif.api.model.predicate.Predicate;
020import org.gbif.api.model.predicate.WithinPredicate;
021import org.gbif.api.vocabulary.Extension;
022
023import java.io.ByteArrayOutputStream;
024import java.io.IOException;
025import java.io.RandomAccessFile;
026import java.util.Collections;
027
028import org.hamcrest.core.IsCollectionContaining;
029import org.junit.jupiter.api.Disabled;
030import org.junit.jupiter.api.Test;
031
032import com.fasterxml.jackson.databind.ObjectMapper;
033
034import static org.hamcrest.CoreMatchers.both;
035import static org.hamcrest.CoreMatchers.equalTo;
036import static org.hamcrest.CoreMatchers.not;
037import static org.hamcrest.MatcherAssert.assertThat;
038import static org.junit.jupiter.api.Assertions.assertEquals;
039import static org.junit.jupiter.api.Assertions.assertFalse;
040import static org.junit.jupiter.api.Assertions.assertNotNull;
041import static org.junit.jupiter.api.Assertions.assertNull;
042import static org.junit.jupiter.api.Assertions.assertTrue;
043import static org.junit.jupiter.api.Assertions.fail;
044import static org.mockito.Mockito.mock;
045
046/**
047 * Test cases for DownloadRequest serialization and building.
048 */
049public class DownloadRequestTest {
050
051  private static final String TEST_USER = "user@gbif.org";
052  private static final String TEST_EMAIL = "test@gbif.org";
053  private static final String TEST_DESCRIPTION = "Unit test";
054
055  // Note these include each combination of underscores or camel case for notificationAddresses and sendNotification.
056
057  private static final String SIMPLE_CSV = "{"
058      + " \"creator\":\"" + TEST_USER + "\", "
059      + " \"notification_addresses\": [\"" + TEST_EMAIL +"\"],"
060      + " \"send_notification\":\"true\","
061      + " \"format\": \"SIMPLE_CSV\","
062      + " \"predicate\":{\"type\":\"equals\",\"key\":\"TAXON_KEY\",\"value\":\"3\"},"
063      + " \"description\": \"" + TEST_DESCRIPTION + "\""
064      + "}";
065
066  private static final String SIMPLE_CSV_NULL_PREDICATE = "{"
067      + " \"creator\":\"" + TEST_USER + "\", "
068      + " \"notificationAddress\": [\"" + TEST_EMAIL +"\"],"
069      + " \"sendNotification\":true," // Note boolean rather than string
070      + " \"format\": \"SIMPLE_CSV\""
071      + "}";
072
073  private static final String SIMPLE_CSV_NULL_PREDICATE_AVAIL = "{"
074      + " \"creator\":\"" + TEST_USER + "\", "
075      + " \"notification_address\": [\"" + TEST_EMAIL +"\"],"
076      + " \"send_notification\":\"true\","
077      + " \"format\": \"SIMPLE_CSV\","
078      + " \"predicate\": null"
079      + "}";
080
081  private static final String SQL_REQUEST = "{"
082    + " \"creator\":\"" + TEST_USER + "\","
083    + " \"notificationAddresses\": [\"" + TEST_EMAIL +"\"],"
084    + " \"sendNotification\":\"true\","
085    + " \"format\": \"SQL_TSV_ZIP\","
086    + " \"sql\": \"SELECT basisOfRecord, COUNT(DISTINCT speciesKey) AS speciesCount FROM occurrence WHERE year = 2018 GROUP BY basisOfRecord\","
087    + " \"description\": \"" + TEST_DESCRIPTION + "\","
088    + " \"machineDescription\": {\"purpose\": \"" + TEST_DESCRIPTION + "\", \"elements\": [], \"array\": [{}, \"a\", 1.0, true, null]}"
089    + "}";
090
091  private static final String UNKNOWN_PROPERTY = "{"
092    + " \"creator\":\"" + TEST_USER + "\", "
093    + " \"notification_addresses\": [\"" + TEST_EMAIL +"\"],"
094    + " \"send_notification\":\"true\","
095    + " \"format\": \"SIMPLE_CSV\","
096    + " \"predicate\":{\"type\":\"equals\",\"key\":\"TAXON_KEY\",\"value\":\"3\"},"
097    + " \"somethingElse\": 12345"
098    + "}";
099
100
101  private static final ObjectMapper MAPPER = new ObjectMapper();
102
103  private static PredicateDownloadRequest newDownload(Predicate p) {
104    return newDownload(p, TEST_USER);
105  }
106
107  private static PredicateDownloadRequest newDownload(Predicate p, String user) {
108    return new PredicateDownloadRequest(
109        p,
110        user,
111        Collections.singleton(TEST_EMAIL),
112        false,
113        DownloadFormat.DWCA,
114        DownloadType.OCCURRENCE,
115        "Unit test download",
116        null,
117        Collections.singleton(Extension.AUDUBON),
118        Collections.singleton(Extension.HUMBOLDT),
119        Constants.NUB_DATASET_KEY.toString());
120  }
121
122  @Test
123  public void testAvailable() {
124    //When a Download is created it has RUNNING as its status
125    Download download = new Download();
126    download.setStatus(Download.Status.RUNNING);
127    assertFalse(download.isAvailable());
128
129    //Status changed
130    download.setStatus(Download.Status.SUCCEEDED);
131    assertTrue(download.isAvailable());
132  }
133
134  @Test
135  public void testBasic() {
136    Predicate p = mock(Predicate.class);
137    PredicateDownloadRequest dl = newDownload(p);
138
139    assertThat(dl.getNotificationAddresses(), IsCollectionContaining.hasItem(TEST_EMAIL));
140    assertThat(dl.getPredicate(), equalTo(p));
141  }
142
143  @Test
144  public void testEquals() {
145    Predicate p = mock(Predicate.class);
146
147    DownloadRequest dl1 = newDownload(p);
148    DownloadRequest dl2 = newDownload(p);
149
150    assertThat(dl1, both(equalTo(dl1)).and(equalTo(dl2)));
151  }
152
153  @Test
154  public void testHashcode() {
155    Predicate p = mock(Predicate.class);
156
157    PredicateDownloadRequest dl1 = newDownload(p);
158    PredicateDownloadRequest dl2 = newDownload(p);
159
160    assertThat(dl1.hashCode(), both(equalTo(dl1.hashCode())).and(equalTo(dl2.hashCode())));
161
162    dl2 = newDownload(p, TEST_EMAIL);
163    assertThat(dl1.hashCode(), not(equalTo(dl2.hashCode())));
164  }
165
166  @Test
167  public void testSerde() throws IOException {
168    PredicateDownloadRequest d = newDownload(new EqualsPredicate(OccurrenceSearchParameter.CATALOG_NUMBER, "b", false));
169    PredicateDownloadRequest d2;
170    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
171      MAPPER.writeValue(baos, d);
172      d2 = MAPPER.readValue(baos.toByteArray(), PredicateDownloadRequest.class);
173    } catch (Throwable e) { // closer must catch Throwable
174      fail(e.getMessage());
175      throw e;
176    }
177
178    assertEquals(d, d2);
179  }
180
181  @Test
182  public void testPredicateDownloadSerde() throws IOException {
183    DownloadRequest request = MAPPER.readValue(SIMPLE_CSV, PredicateDownloadRequest.class);
184    assertEquals(TEST_USER, request.getCreator());
185    assertEquals(DownloadFormat.SIMPLE_CSV, request.getFormat());
186    assertTrue(request.getSendNotification());
187    assertEquals(TEST_EMAIL, request.getNotificationAddressesAsString());
188    assertEquals(TEST_DESCRIPTION, request.getDescription());
189  }
190
191  @Test
192  public void testDownloadRequestSerde() throws IOException {
193    DownloadRequest request = MAPPER.readValue(SIMPLE_CSV, DownloadRequest.class);
194    assertEquals(TEST_USER, request.getCreator());
195    assertEquals(DownloadFormat.SIMPLE_CSV, request.getFormat());
196    assertTrue(request.getSendNotification());
197    assertEquals(TEST_EMAIL, request.getNotificationAddressesAsString());
198    assertEquals(TEST_DESCRIPTION, request.getDescription());
199  }
200
201  @Disabled
202  @Test
203  public void testDownloadRequestMissingPredicate() throws IOException {
204    try {
205      DownloadRequest request = MAPPER.readValue(SIMPLE_CSV_NULL_PREDICATE, DownloadRequest.class);
206      fail();
207    } catch (Exception e) {
208      assertEquals("A predicate must be specified. Use {} for everything.", e.getMessage());
209    }
210  }
211
212  @Test
213  public void testDownloadRequestSerde2() throws IOException {
214    DownloadRequest request = MAPPER.readValue(SIMPLE_CSV_NULL_PREDICATE_AVAIL, DownloadRequest.class);
215    assertNull(((PredicateDownloadRequest)request).getPredicate());
216    assertTrue(request.getSendNotification());
217    assertEquals(TEST_EMAIL, request.getNotificationAddressesAsString());
218  }
219
220  @Test
221  public void testSQLDownloadSerde() throws IOException {
222    SqlDownloadRequest request = MAPPER.readValue(SQL_REQUEST, SqlDownloadRequest.class);
223    assertEquals(TEST_USER, request.getCreator());
224    assertNotNull(request.getSql());
225    assertEquals(DownloadFormat.SQL_TSV_ZIP, request.getFormat());
226    assertTrue(request.getSendNotification());
227    assertEquals(TEST_EMAIL, request.getNotificationAddressesAsString());
228    assertEquals(TEST_DESCRIPTION, request.getDescription());
229
230    // Test no-schema JSON
231    assertEquals(TEST_DESCRIPTION, request.getMachineDescription().get("purpose").asText());
232    assertEquals(0, request.getMachineDescription().get("elements").size());
233    assertEquals(0, request.getMachineDescription().get("array").get(0).size());
234    assertEquals("a", request.getMachineDescription().get("array").get(1).asText());
235    assertEquals("1.0", request.getMachineDescription().get("array").get(2).asText());
236    assertEquals(1, request.getMachineDescription().get("array").get(2).asInt());
237    assertTrue(request.getMachineDescription().get("array").get(3).asBoolean());
238    assertTrue(request.getMachineDescription().get("array").get(4).isNull());
239  }
240
241  /**
242   * Tests that an unknown property is rejected.
243   *
244   * For much of the API these are allowed, but it causes problems when typing errors etc trigger all-data downloads.
245   */
246  @Test
247  public void testDownloadRequestSerdeUnknown() throws IOException {
248    try {
249      DownloadRequest request = MAPPER.readValue(UNKNOWN_PROPERTY, DownloadRequest.class);
250      fail();
251    } catch (Exception e) {
252      assertEquals("Unknown JSON property 'somethingElse'.", e.getMessage());
253    }
254  }
255
256  /**
257   * Tests that an empty JSON {} is translated into null.
258   */
259  @Test
260  public void testDownloadRequestSerdeNull() throws IOException {
261    String json = "{}";
262    DownloadRequest request = MAPPER.readValue(json, DownloadRequest.class);
263    assertNull(request);
264  }
265
266  /**
267   * Request sent by PyGBIF ≤ 0.6.1.  For backward compatibility, do not change this test!
268   *
269   * Note the 'created' property, the quoted booleans, and the underscore in notification_address.
270   */
271  @Test
272  public void downloadFromPygbifTest() throws Exception {
273    String downloadRequest = "{\n"
274      + "  \"creator\":\"pygbif\",\n"
275      + "  \"notification_address\":[\"pygbif@mailinator.com\"],\n"
276      + "  \"created\":\"2023\",\n"
277      + "  \"sendNotification\":\"true\",\n"
278      + "  \"format\": \"SIMPLE_CSV\",\n"
279      + "  \"predicate\":{\n"
280      + "    \"type\":\"within\",\n"
281      + "    \"geometry\":\"POLYGON ((-85.781 17.978,-81.035 14.774,-77.343 10.314,-79.277 6.315,-93.955 14.604,-91.450 18.229,-87.626 19.311,-85.781 17.978))\"\n"
282      + "  }\n"
283      + "}";
284
285    DownloadRequest request = MAPPER.readValue(downloadRequest, DownloadRequest.class);
286    assertEquals("pygbif", request.getCreator());
287    assertEquals(DownloadFormat.SIMPLE_CSV, request.getFormat());
288    assertTrue(request.getSendNotification());
289    assertEquals("pygbif@mailinator.com", request.getNotificationAddressesAsString());
290    assertEquals(WithinPredicate.class, ((PredicateDownloadRequest) request).getPredicate().getClass());
291  }
292
293  /**
294   * Request sent by RGBIF 3.75+.  For backward compatibility, do not change this test!
295   *
296   * Note the lack of sendNotification.
297   */
298  @Test
299  public void downloadFromRgbifTest() throws Exception {
300    String downloadRequest = "{\n"
301      + "  \"creator\":\"rgbif\",\n"
302      + "  \"notification_address\":[\"rgbif@mailinator.com\"],\n"
303      + "  \"format\": \"SIMPLE_CSV\",\n"
304      + "  \"predicate\":{\n"
305      + "    \"type\":\"within\",\n"
306      + "    \"geometry\":\"POLYGON ((-85.781 17.978,-81.035 14.774,-77.343 10.314,-79.277 6.315,-93.955 14.604,-91.450 18.229,-87.626 19.311,-85.781 17.978))\"\n"
307      + "  }\n"
308      + "}";
309
310    DownloadRequest request = MAPPER.readValue(downloadRequest, DownloadRequest.class);
311    assertEquals("rgbif", request.getCreator());
312    assertEquals(DownloadFormat.SIMPLE_CSV, request.getFormat());
313    assertFalse(request.getSendNotification());
314    assertEquals("rgbif@mailinator.com", request.getNotificationAddressesAsString());
315    assertEquals(WithinPredicate.class, ((PredicateDownloadRequest) request).getPredicate().getClass());
316  }
317
318  /**
319   * Working request at v0.188.  For backward compatibility, do not change this test!
320   *
321   * Note the 212 rather than "212".
322   */
323  @Test
324  public void downloadWithNumbersTest() throws Exception {
325    String downloadRequest = "{\n"
326      + "  \"creator\":\"gbif_user\",\n"
327      + "  \"notification_address\":[\"gbif_user@mailinator.com\"],\n"
328      + "  \"created\":\"2023\",\n"
329      + "  \"format\": \"SIMPLE_CSV\",\n"
330      + "  \"predicate\":{\n"
331      + "    \"type\":\"equals\",\n"
332      + "    \"key\":\"TAXON_KEY\",\n"
333      + "    \"value\":212\n"
334      + "  }\n"
335      + "}";
336
337    DownloadRequest request = MAPPER.readValue(downloadRequest, DownloadRequest.class);
338    assertEquals("gbif_user", request.getCreator());
339    assertEquals(DownloadFormat.SIMPLE_CSV, request.getFormat());
340    assertFalse(request.getSendNotification());
341    assertEquals("gbif_user@mailinator.com", request.getNotificationAddressesAsString());
342    assertEquals(EqualsPredicate.class, ((PredicateDownloadRequest) request).getPredicate().getClass());
343    assertEquals("212", ((EqualsPredicate) ((PredicateDownloadRequest) request).getPredicate()).getValue());
344  }
345
346  /**
347   * Test three extension situations: known and supported, known but not supported, unknown.
348   */
349  @Test
350  public void downloadWithExtensionTest() throws Exception {
351    String requestTemplate = "{\n"
352      + "  \"creator\":\"gbif_user\",\n"
353      + "  \"notification_address\":[\"gbif_user@mailinator.com\"],\n"
354      + "  \"created\":\"2024\",\n"
355      + "  \"format\": \"DWCA\",\n"
356      + "  \"predicate\":{\n"
357      + "    \"type\":\"equals\",\n"
358      + "    \"key\":\"TAXON_KEY\",\n"
359      + "    \"value\":\"212\"\n"
360      + "  },\n"
361      + "  %s\n"
362      + "}";
363
364    DownloadRequest request;
365
366    // Known and supported extension
367    request = MAPPER.readValue(String.format(requestTemplate, "'verbatimExtensions':['http://rs.tdwg.org/dwc/terms/MeasurementOrFact']".replace("'", "\"")), DownloadRequest.class);
368    assertEquals("gbif_user", request.getCreator());
369    assertEquals(DownloadFormat.DWCA, request.getFormat());
370    assertEquals("gbif_user@mailinator.com", request.getNotificationAddressesAsString());
371    assertEquals(EqualsPredicate.class, ((PredicateDownloadRequest) request).getPredicate().getClass());
372    assertEquals("212", ((EqualsPredicate) ((PredicateDownloadRequest) request).getPredicate()).getValue());
373    assertEquals(Extension.MEASUREMENT_OR_FACT, ((PredicateDownloadRequest) request).getVerbatimExtensions().iterator().next());
374
375    // Known and *unsupported* extension
376    try {
377      MAPPER.readValue(String.format(requestTemplate, "'verbatimExtensions':['http://zooarchnet.org/dwc/terms/ChronometricDate']".replace("'", "\"")), DownloadRequest.class);
378      fail();
379    } catch (Exception e) {
380    }
381
382    // Unknown extension
383    try {
384      MAPPER.readValue(String.format(requestTemplate, "'verbatimExtensions':['http://example.org/nothing']".replace("'", "\"")), DownloadRequest.class);
385      fail();
386    } catch (Exception e) {
387    }
388
389    // Extension enum value — not supported as these are a bit too internal
390    try {
391      MAPPER.readValue(String.format(requestTemplate, "'verbatimExtensions':['MEASUREMENT_OR_FACT']".replace("'", "\"")), DownloadRequest.class);
392      fail();
393    } catch (Exception e) {
394    }
395
396    // Null value
397    try {
398      MAPPER.readValue(String.format(requestTemplate, "'verbatimExtensions':[null]".replace("'", "\"")), DownloadRequest.class);
399      fail();
400    } catch (Exception e) {
401    }
402  }
403
404  @Disabled
405  @Test
406  public void existingDownloadsTest() throws Exception {
407    String format = "{\"predicate\": %s}";
408    String line = null;
409    String[] filterLine = null;
410    // SELECT key, filter FROM occurrence_download WHERE filter IS NOT NULL ORDER BY created
411    try (RandomAccessFile filterLines = new RandomAccessFile("/home/mblissett/Temp/all-download-filters", "r")) {
412      while ((line = filterLines.readLine()) != null ) {
413        filterLine = line.split("\t");
414        MAPPER.readValue(String.format(format, filterLine[1]), DownloadRequest.class);
415      }
416    } catch (Exception e) {
417      System.err.println(e);
418      fail("Exception with existing download " + filterLine[0] + " «" + filterLine[1] + "»");
419    }
420  }
421}