Merge pull request #53 from lfoppiano/sofair

Contribution and changes from the SoFAIR project

Author: jameshowison

build.gradle

@@ -362,7 +362,7 @@ task downloadModelsGit {
         } else {
             def grgit = Grgit.clone(
                 dir: "${grobidHome}/models/softcite",
-                uri: "https://huggingface.co/sciencialab/software-mentions-models",
+                uri: "https://huggingface.co/SoFairOA/software-mentions-models",
                 depth: 1
             )
         }

localLibs/org/grobid/grobid-service/0.8.2/grobid-service-0.8.2.pom

@@ -269,7 +269,7 @@
     <dependency>
       <groupId>org.grobid</groupId>
       <artifactId>grobid-core</artifactId>
-      <version>0.8.3-SNAPSHOT</version>
+      <version>0.8.2</version>
       <scope>runtime</scope>
       <exclusions>
         <exclusion>
@@ -285,7 +285,7 @@
     <dependency>
       <groupId>org.grobid</groupId>
       <artifactId>grobid-trainer</artifactId>
-      <version>0.8.3-SNAPSHOT</version>
+      <version>0.8.2</version>
       <scope>runtime</scope>
       <exclusions>
         <exclusion>

src/main/java/org/grobid/core/engines/SoftwareParser.java

@@ -879,10 +879,10 @@ private List<SoftwareEntity> processLayoutTokenSequenceMultiple(
      */
     private List<SoftwareEntity> processLayoutTokenSequences(
         List<LayoutTokenization> layoutTokenizations,
-                                                             List<SoftwareEntity> entities,
-                                                             boolean disambiguate,
-                                                             boolean addParagraphContext,
-                                                             boolean fromPDF,
+        List<SoftwareEntity> entities,
+        boolean disambiguate,
+        boolean addParagraphContext,
+        boolean fromPDF,
         boolean fromXML,
         List<PDFAnnotation> pdfAnnotations
     ) {
@@ -896,12 +896,14 @@ private List<SoftwareEntity> processLayoutTokenSequences(
 
             // positions for lexical match
             List<OffsetPosition> softwareTokenPositions = softwareLexicon.tokenPositionsSoftwareNames(layoutTokens);
-            List<OffsetPosition> urlPositions = Lexicon.tokenPositionUrlPatternWithPdfAnnotations(layoutTokens, pdfAnnotations).stream()
+            List<OffsetPosition> urlTokensPositions = Lexicon.tokenPositionUrlPatternWithPdfAnnotations(layoutTokens, pdfAnnotations).stream()
                 .map(Pair::getLeft)
                 .collect(Collectors.toList());
 
+            urlTokensPositions.stream().forEach(o -> o.end += 1);
+
             // string representation of the feature matrix for sequence labeling lib
-            String ress = addFeatures(layoutTokens, softwareTokenPositions, urlPositions);
+            String ress = addFeatures(layoutTokens, softwareTokenPositions, urlTokensPositions);
             allRess.append(ress);
             allRess.append("\n\n");
         }
@@ -990,10 +992,13 @@ private List<SoftwareEntity> processLayoutTokenSequences(
      * Process with the software model a set of arbitrary sequence of LayoutTokenization
      * from tables and figures, where the content is not structured (yet)
      */
-    private List<SoftwareEntity> processLayoutTokenSequenceTableFigure(List<LayoutToken> layoutTokens,
-                                                                       List<SoftwareEntity> entities,
-                                                                       boolean disambiguate,
-                                                                       boolean addParagraphContext) {
+    private List<SoftwareEntity> processLayoutTokenSequenceTableFigure(
+        List<LayoutToken> layoutTokens,
+        List<SoftwareEntity> entities,
+        boolean disambiguate,
+        boolean addParagraphContext,
+        List<PDFAnnotation> pdfAnnotations
+    ) {
         layoutTokens = SoftwareAnalyzer.getInstance().retokenizeLayoutTokens(layoutTokens);
 
         int pos = 0;
@@ -1016,7 +1021,9 @@ private List<SoftwareEntity> processLayoutTokenSequenceTableFigure(List<LayoutTo
 
             // positions for lexical match
             List<OffsetPosition> softwareTokenPositions = softwareLexicon.tokenPositionsSoftwareNames(localLayoutTokens);
-            List<OffsetPosition> urlPositions = Lexicon.getInstance().tokenPositionsUrlPattern(localLayoutTokens);
+            List<OffsetPosition> urlPositions = Lexicon.tokenPositionUrlPatternWithPdfAnnotations(layoutTokens, pdfAnnotations).stream()
+                .map(Pair::getLeft)
+                .collect(Collectors.toList());
 
             // string representation of the feature matrix for sequence labeling lib
             String ress = addFeatures(localLayoutTokens, softwareTokenPositions, urlPositions);