Add oaDOI fulltext fetcher (#3581)

tobiasdiez · web-flow · commit b246b9ce534a · 2018-01-02T22:22:57.000+01:00
* Add oaDOI fulltext fetcher

* Rename to OpenAccessDoi
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 ## [Unreleased]
 
 ### Changed
+- We added [oaDOI](https://oadoi.org/) as a fulltext provider, so that JabRef is now able to provide fulltexts for more than 90 million open-access articles.
 
 
 ### Fixed
diff --git a/src/main/java/org/jabref/logic/importer/FulltextFetchers.java b/src/main/java/org/jabref/logic/importer/FulltextFetchers.java
@@ -6,13 +6,6 @@
 import java.util.List;
 import java.util.Optional;
 
-import org.jabref.logic.importer.fetcher.ACS;
-import org.jabref.logic.importer.fetcher.ArXiv;
-import org.jabref.logic.importer.fetcher.DoiResolution;
-import org.jabref.logic.importer.fetcher.GoogleScholar;
-import org.jabref.logic.importer.fetcher.IEEE;
-import org.jabref.logic.importer.fetcher.ScienceDirect;
-import org.jabref.logic.importer.fetcher.SpringerLink;
 import org.jabref.logic.net.URLDownload;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.FieldName;
@@ -30,26 +23,13 @@ public class FulltextFetchers {
     private final List<FulltextFetcher> finders = new ArrayList<>();
 
     public FulltextFetchers(ImportFormatPreferences importFormatPreferences) {
-        // Ordering is important, authorities first!
-        // Publisher
-        finders.add(new DoiResolution());
-        finders.add(new ScienceDirect());
-        finders.add(new SpringerLink());
-        finders.add(new ACS());
-        finders.add(new ArXiv(importFormatPreferences));
-        finders.add(new IEEE());
-        // Meta search
-        finders.add(new GoogleScholar(importFormatPreferences));
+        this(WebFetchers.getFullTextFetchers(importFormatPreferences));
     }
 
-    public FulltextFetchers(List<FulltextFetcher> fetcher) {
+    FulltextFetchers(List<FulltextFetcher> fetcher) {
         finders.addAll(fetcher);
     }
 
-    public List<FulltextFetcher> getFetchers() {
-        return finders;
-    }
-
     public Optional<URL> findFullTextPDF(BibEntry entry) {
         // for accuracy, fetch DOI first but do not modify entry
         BibEntry clonedEntry = (BibEntry) entry.clone();
diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java
@@ -5,19 +5,25 @@
 import java.util.List;
 import java.util.Optional;
 
+import org.jabref.logic.importer.fetcher.ACS;
 import org.jabref.logic.importer.fetcher.ArXiv;
 import org.jabref.logic.importer.fetcher.AstrophysicsDataSystem;
 import org.jabref.logic.importer.fetcher.CrossRef;
 import org.jabref.logic.importer.fetcher.DBLPFetcher;
 import org.jabref.logic.importer.fetcher.DiVA;
 import org.jabref.logic.importer.fetcher.DoiFetcher;
+import org.jabref.logic.importer.fetcher.DoiResolution;
 import org.jabref.logic.importer.fetcher.GoogleScholar;
 import org.jabref.logic.importer.fetcher.GvkFetcher;
+import org.jabref.logic.importer.fetcher.IEEE;
 import org.jabref.logic.importer.fetcher.IacrEprintFetcher;
 import org.jabref.logic.importer.fetcher.IsbnFetcher;
 import org.jabref.logic.importer.fetcher.LibraryOfCongress;
 import org.jabref.logic.importer.fetcher.MathSciNet;
 import org.jabref.logic.importer.fetcher.MedlineFetcher;
+import org.jabref.logic.importer.fetcher.OpenAccessDoi;
+import org.jabref.logic.importer.fetcher.ScienceDirect;
+import org.jabref.logic.importer.fetcher.SpringerLink;
 import org.jabref.logic.importer.fetcher.TitleFetcher;
 import org.jabref.logic.importer.fetcher.zbMATH;
 import org.jabref.model.entry.FieldName;
@@ -113,4 +119,22 @@ public static List<IdFetcher> getIdFetchers(ImportFormatPreferences importFormat
         list.sort(Comparator.comparing(WebFetcher::getName));
         return list;
     }
+
+    public static List<FulltextFetcher> getFullTextFetchers(ImportFormatPreferences importFormatPreferences) {
+        List<FulltextFetcher> fetchers = new ArrayList<>();
+
+        // Ordering is important, authorities first!
+        // Publisher
+        fetchers.add(new DoiResolution());
+        fetchers.add(new ScienceDirect());
+        fetchers.add(new SpringerLink());
+        fetchers.add(new ACS());
+        fetchers.add(new ArXiv(importFormatPreferences));
+        fetchers.add(new IEEE());
+        // Meta search
+        fetchers.add(new GoogleScholar(importFormatPreferences));
+        fetchers.add(new OpenAccessDoi());
+
+        return fetchers;
+    }
 }
diff --git a/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java b/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java
@@ -0,0 +1,58 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.jabref.logic.importer.FulltextFetcher;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.FieldName;
+import org.jabref.model.entry.identifier.DOI;
+
+import com.mashape.unirest.http.HttpResponse;
+import com.mashape.unirest.http.JsonNode;
+import com.mashape.unirest.http.Unirest;
+import com.mashape.unirest.http.exceptions.UnirestException;
+import org.json.JSONObject;
+
+/**
+ * A fulltext fetcher that uses <a href="https://oadoi.org/">oaDOI</a>.
+ *
+ * @implSpec API is documented at https://oadoi.org/api/v2
+ */
+public class OpenAccessDoi implements FulltextFetcher {
+    private static String API_URL = "https://api.oadoi.org/v2/";
+
+    @Override
+    public Optional<URL> findFullText(BibEntry entry) throws IOException {
+        Objects.requireNonNull(entry);
+
+        Optional<DOI> doi = entry.getField(FieldName.DOI)
+                .flatMap(DOI::parse);
+        if (doi.isPresent()) {
+            try {
+                return findFullText(doi.get());
+            } catch (UnirestException e) {
+                throw new IOException(e);
+            }
+        } else {
+            return Optional.empty();
+        }
+    }
+
+    public Optional<URL> findFullText(DOI doi) throws UnirestException, MalformedURLException {
+        HttpResponse<JsonNode> jsonResponse = Unirest.get(API_URL + doi.getDOI() + "?email=developers@jabref.org")
+                .header("accept", "application/json")
+                .asJson();
+        JSONObject root = jsonResponse.getBody().getObject();
+        Optional<String> url = Optional.ofNullable(root.optJSONObject("best_oa_location"))
+                .map(location -> location.optString("url"));
+        if (url.isPresent()) {
+            return Optional.of(new URL(url.get()));
+        } else {
+            return Optional.empty();
+        }
+    }
+}
diff --git a/src/main/java/org/jabref/model/entry/identifier/DOI.java b/src/main/java/org/jabref/model/entry/identifier/DOI.java
@@ -95,7 +95,7 @@ public DOI(String doi) {
      */
     public static Optional<DOI> parse(String doi) {
         try {
-            return Optional.ofNullable(new DOI(doi));
+            return Optional.of(new DOI(doi));
         } catch (IllegalArgumentException | NullPointerException e) {
             return Optional.empty();
         }
diff --git a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java
@@ -51,11 +51,19 @@ public void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher(
 
     @Test
     public void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() throws Exception {
-        List<SearchBasedFetcher> idFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences);
+        List<SearchBasedFetcher> searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences);
 
         Set<Class<? extends SearchBasedFetcher>> expected = reflections.getSubTypesOf(SearchBasedFetcher.class);
         expected.remove(SearchBasedParserFetcher.class);
-        assertEquals(expected, getClasses(idFetchers));
+        assertEquals(expected, getClasses(searchBasedFetchers));
+    }
+
+    @Test
+    public void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() throws Exception {
+        List<FulltextFetcher> fullTextFetchers = WebFetchers.getFullTextFetchers(importFormatPreferences);
+
+        Set<Class<? extends FulltextFetcher>> expected = reflections.getSubTypesOf(FulltextFetcher.class);
+        assertEquals(expected, getClasses(fullTextFetchers));
     }
 
     @Test
diff --git a/src/test/java/org/jabref/logic/importer/fetcher/FulltextFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/FulltextFetcherTest.java
@@ -4,8 +4,8 @@
 import java.util.Optional;
 
 import org.jabref.logic.importer.FulltextFetcher;
-import org.jabref.logic.importer.FulltextFetchers;
 import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.WebFetchers;
 import org.jabref.model.entry.BibEntry;
 
 import org.junit.jupiter.params.ParameterizedTest;
@@ -18,7 +18,7 @@
 class FulltextFetcherTest {
 
     private static List<FulltextFetcher> fetcherProvider() {
-        return new FulltextFetchers(mock(ImportFormatPreferences.class)).getFetchers();
+        return WebFetchers.getFullTextFetchers(mock(ImportFormatPreferences.class));
     }
 
     @ParameterizedTest
diff --git a/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java b/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java
@@ -0,0 +1,43 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Optional;
+
+import org.jabref.model.entry.BibEntry;
+import org.jabref.testutils.category.FetcherTest;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@FetcherTest
+class OpenAccessDoiTest {
+
+    private OpenAccessDoi finder;
+    private BibEntry entry;
+
+    @BeforeEach
+    void setUp() {
+        finder = new OpenAccessDoi();
+        entry = new BibEntry();
+    }
+
+    @Test
+    void findByDOI() throws IOException {
+        entry.setField("doi", "10.1038/nature12373");
+
+        assertEquals(
+                Optional.of(new URL("https://dash.harvard.edu/bitstream/handle/1/12285462/Nanometer-Scale%20Thermometry.pdf?sequence=1")),
+                finder.findFullText(entry)
+        );
+    }
+
+    @Test
+    void notFoundByDOI() throws IOException {
+        entry.setField("doi", "10.1186/unknown-doi");
+
+        assertEquals(Optional.empty(), finder.findFullText(entry));
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ public DOI(String doi) {`
`95`	`95`	`*/`
`96`	`96`	`public static Optional<DOI> parse(String doi) {`
`97`	`97`	`try {`
`98`		`- return Optional.ofNullable(new DOI(doi));`
	`98`	`+ return Optional.of(new DOI(doi));`
`99`	`99`	`} catch (IllegalArgumentException \| NullPointerException e) {`
`100`	`100`	`return Optional.empty();`
`101`	`101`	`}`