001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 *     http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014package org.gbif.dwc;
015
016import org.gbif.dwc.record.StarRecord;
017
018import java.net.URL;
019import java.nio.file.Files;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022
023import org.slf4j.Logger;
024import org.slf4j.LoggerFactory;
025
026import static org.junit.jupiter.api.Assertions.assertFalse;
027import static org.junit.jupiter.api.Assertions.assertTrue;
028
029/**
030 * This test checks we don't get a race condition between two <strong>processes</strong> opening the same DWC-A at the
031 * same time.
032 *
033 * It needs to be run from the command line, in parallel.
034 *
035 * mvn exec:java -Dexec.mainClass=org.gbif.dwc.ParallelOpeningTest
036 *
037 * Monitor open files with something like:
038 *
039 * while :; do for i in `pgrep -f ParallelOpeningTest`; do echo $i && ls -l /proc/$i/fd | grep /tmp/par && echo; sleep 1; done; done
040 */
041public class ParallelOpeningTest {
042  private static final Logger LOG = LoggerFactory.getLogger(ParallelOpeningTest.class);
043
044  /**
045   * Check we don't get a race condition when opening an archive which needs sorting.
046   *
047   * This downloads a large archive (we don't want it in Git) and checks it has at least one extension.
048   */
049  public void testParallelOpening() throws Exception {
050    Path archivePath = Paths.get("/tmp/parallel-opening-test-archive.zip");
051    Path extractToFolder = Paths.get("/tmp/parallel-opening-test-archive");
052    Archive arch;
053    if (!archivePath.toFile().exists()) {
054      URL download = new URL("http://api.gbif.org/v1/occurrence/download/request/0012957-180131172636756.zip");
055      Files.copy(download.openStream(), archivePath);
056      extractToFolder.toFile().mkdir();
057      arch = DwcFiles.fromCompressed(archivePath, extractToFolder);
058    }
059
060    arch = DwcFiles.fromLocation(extractToFolder);
061
062    assertFalse(arch.getExtensions().isEmpty());
063
064    // Initialize (i.e. normalize and sort) the archive in the background.
065    LOG.info("Initializing archive");
066    arch.initialize();
067    LOG.info("Initialization completed, locks should have been released.");
068
069    int counter = 0;
070    for (StarRecord rec : arch) {
071      if (counter == 0) {
072        LOG.info("Read first record {}", rec);
073      }
074      counter++;
075    }
076
077    LOG.info("Counted {} records", counter);
078
079    LOG.info("All files should have been closed.");
080    Thread.sleep(30_000);
081
082    assertTrue(counter > 500000, "Many records extracted");
083  }
084
085  public static void main(String... args) throws Exception {
086    ParallelOpeningTest pot = new ParallelOpeningTest();
087    pot.testParallelOpening();
088  }
089}