Merge pull request #250 from simleo/detached_crates

Support for detached crates

Author: simleo

README.md

@@ -293,6 +293,68 @@ article = crate.dereference("paper.pdf")
 
 ## Advanced features
 
+### Detached crates
+
+[RO-Crate 1.2](https://www.researchobject.org/ro-crate/whats-changed-in-1-2) introduces the concept of _detached_ RO-Crates, which have no defined root directory: in detached crates, the metadata is accessed independently, for instance via an API or from a standalone metadata file. By contrast, "traditional" crates that describe a payload of files and directories contained in a root directory are called _attached_.
+
+Both detached and attached crates can have a root data entity with an absolute URI as `@id`. To create an RO-Crate whose root data entity `@id` is different from the default `./`, use the `root_dataset_id` argument in the constructor:
+
+```python
+from rocrate.rocrate import ROCrate
+
+url = "http://example.com/crate/"
+crate = ROCrate(root_dataset_id=url)
+```
+
+In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`:
+
+```python
+file_1 = crate.add_file(f"{url}file_1")  # http://example.com/crate/file_1
+```
+
+The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`, where `${prefix}` is a variable. The library supports this through the `write_detached` method, which takes as argument an arbitrary path (a warning will be issued if the path does not follow the above pattern):
+
+```python
+crate.write_detached("/tmp/example-ro-crate-metadata.json")
+```
+
+One of the ways to consume a detached crate is to read the metadata from a local file. For instance, to read the crate that we just wrote:
+
+```python
+read_crate = ROCrate("/tmp/example-ro-crate-metadata.json")
+read_file_1 = read_crate.dereference(f"{url}file_1")
+```
+
+This also works with a local `file://` URI:
+
+```python
+read_crate = ROCrate("file:///tmp/example-ro-crate-metadata.json")
+```
+
+and with a remote URI:
+
+```python
+base = "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/"
+read_crate = ROCrate(f"{base}detached-ro-crate-metadata.json")
+assert read_crate.root_dataset.id == base
+sample_file = read_crate.dereference(f"{base}sample_file.txt")
+test_file_galaxy = read_crate.dereference(f"{base}test_file_galaxy.txt")
+```
+
+Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance:
+
+```python
+import json
+from rocrate.rocrate import ROCrate
+
+with open("/tmp/example-ro-crate-metadata.json") as f:
+    metadata = json.load(f)
+crate = ROCrate(metadata)
+```
+
+In the above example we read the metadata from a local file, but you could get it from an API endpoint or any other source.
+
+
 ### Subcrates
 
 An RO-Crate can contain one or more nested RO-Crates. For instance, consider the following layout:

rocrate/metadata.py

@@ -22,9 +22,20 @@
 # limitations under the License.
 
 import json
+import re
 import warnings
+import urllib.request
+
+import requests
 
 from .model.metadata import BASENAME, LEGACY_BASENAME
+from .utils import is_url
+
+# https://www.researchobject.org/ro-crate/specification/1.2/structure
+#   "If stored in a file... the filename SHOULD be..."
+# https://www.researchobject.org/ro-crate/specification/1.2/data-entities
+#   "It is NOT RECOMMENDED to resolve a relative root identifier..."
+MD_PATTERN = re.compile(r".*[/-]ro-crate-metadata.json(ld)?$")
 
 
 def read_metadata(metadata_path):
@@ -36,6 +47,16 @@ def read_metadata(metadata_path):
     """
     if isinstance(metadata_path, dict):
         metadata = metadata_path
+    elif is_url(str(metadata_path)):
+        if not MD_PATTERN.match(metadata_path):
+            warnings.warn(f"URI {metadata_path} should follow the pattern {MD_PATTERN.pattern!r}")
+        if metadata_path.startswith("file:"):
+            with urllib.request.urlopen(metadata_path) as resp:
+                metadata = json.load(resp)
+        else:
+            with requests.get(metadata_path) as resp:
+                resp.raise_for_status()
+                metadata = resp.json()
     else:
         with open(metadata_path, 'r', encoding='utf-8') as f:
             metadata = json.load(f)
@@ -69,48 +90,8 @@ def find_root_entity_id(entities):
     Return a tuple of the corresponding identifiers (descriptor, root).
     If the entities are not found, raise KeyError. If they are found,
     but they don't satisfy the required constraints, raise ValueError.
-
-    In the general case, the metadata file descriptor id can be an
-    absolute URI whose last path segment is "ro-crate-metadata.json[ld]".
-    Since there can be more than one such id in the crate, we need to
-    choose among the corresponding (descriptor, root) entity pairs. First, we
-    exclude those that don't satisfy other constraints, such as the
-    descriptor entity being of type CreativeWork, etc.; if this doesn't
-    leave us with a single pair, we try to pick one with a
-    heuristic. Suppose we are left with the (m1, r1) and (m2, r2) pairs:
-    if r1 is the actual root of this crate, then m2 and r2 are regular
-    files in it, and as such they must appear in r1's hasPart; r2,
-    however, is not required to have a hasPart property listing other
-    files. Thus, we look for a pair whose root entity "contains" all
-    descriptor entities from other pairs. If there is no such pair, or there
-    is more than one, we just return an arbitrary pair.
-
     """
     descriptor = entities.get(BASENAME, entities.get(LEGACY_BASENAME))
-    if descriptor:
-        return _check_descriptor(descriptor, entities)
-    candidates = []
-    for id_, e in entities.items():
-        basename = id_.rsplit("/", 1)[-1]
-        if basename == BASENAME or basename == LEGACY_BASENAME:
-            try:
-                candidates.append(_check_descriptor(e, entities))
-            except ValueError:
-                pass
-    if not candidates:
+    if not descriptor:
         raise KeyError("Metadata file descriptor not found")
-    elif len(candidates) == 1:
-        return candidates[0]
-    else:
-        warnings.warn("Multiple metadata file descriptors, will pick one with a heuristic")
-        descriptor_ids = set(_[0] for _ in candidates)
-        for m_id, r_id in candidates:
-            try:
-                root = entities[r_id]
-                part_ids = set(_["@id"] for _ in root["hasPart"])
-            except KeyError:
-                continue
-            if part_ids >= descriptor_ids - {m_id}:
-                # if True for more than one candidate, this pick is arbitrary
-                return m_id, r_id
-        return candidates[0]  # fall back to arbitrary pick
+    return _check_descriptor(descriptor, entities)

rocrate/model/dataset.py

@@ -31,7 +31,7 @@
 from urllib.parse import unquote
 
 from .file_or_dir import FileOrDir
-from ..utils import is_url, iso_now, Mode
+from ..utils import as_list, is_url, iso_now, Mode
 
 
 class Dataset(FileOrDir):
@@ -122,17 +122,32 @@ def _stream_folder_from_url(self, chunk_size=8192):
                 with urlopen(self.source) as _:
                     self._jsonld['sdDatePublished'] = iso_now()
         else:
-            base = self.source.rstrip("/")
+            if is_url(self.id):
+                relative_dest_uri = self.get("localPath") or self.id
+            else:
+                relative_dest_uri = self.id
+            if is_url(relative_dest_uri):
+                if relative_dest_uri.startswith(self.crate.root_dataset.id):
+                    relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):]
+                else:
+                    relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1]
+            out_dir_path = Path(unquote(relative_dest_uri))
+
             for entry in self._jsonld.get("hasPart", []):
                 try:
                     part = entry["@id"]
-                    if is_url(part) or part.startswith("/"):
-                        raise RuntimeError(f"'{self.source}': part '{part}' is not a relative path")
-                    part_uri = f"{base}/{part}"
-                    rel_out_path = Path(self.id) / part
-
+                    if not is_url(part):
+                        raise RuntimeError(f"'{self.source}' is a URL, but part '{part}' is not a URL")
+                    rel_out_path = out_dir_path / part.rsplit("/", 1)[-1]
+                    if part_entity := self.crate.get(part):
+                        if "Dataset" in as_list(part_entity.type):
+                            continue
+                        # override with file localPath if set
+                        if "File" in as_list(part_entity.type):
+                            if file_local_path := part_entity.get("localPath"):
+                                rel_out_path = file_local_path
                     is_empty = True
-                    with urlopen(part_uri) as response:
+                    with urlopen(part) as response:
                         while chunk := response.read(chunk_size):
                             is_empty = False
                             yield str(rel_out_path), chunk

rocrate/model/file.py

@@ -72,7 +72,19 @@ def _copy_file(self, path, out_file_path):
             self._jsonld['contentSize'] = str(out_file_path.stat().st_size)
 
     def write(self, base_path):
-        out_file_path = Path(base_path) / unquote(self.id)
+        if self.fetch_remote and is_url(str(self.source)):
+            if is_url(self.id):
+                relative_dest_uri = self.get("localPath") or self.id
+            else:
+                relative_dest_uri = self.id
+            if is_url(relative_dest_uri):
+                if relative_dest_uri.startswith(self.crate.root_dataset.id):
+                    relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):]
+                else:
+                    relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1]
+        else:
+            relative_dest_uri = self.id
+        out_file_path = Path(base_path) / unquote(relative_dest_uri)
         if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)):
             self._write_from_stream(out_file_path)
         elif self.source is None:

rocrate/model/metadata.py

@@ -25,6 +25,8 @@
 
 import json
 from pathlib import Path
+import re
+import warnings
 
 from .file import File
 from .dataset import Dataset
@@ -38,6 +40,7 @@
 DEFAULT_VERSION = "1.2"
 BASENAME = "ro-crate-metadata.json"
 LEGACY_BASENAME = "ro-crate-metadata.jsonld"
+DETACHED_MD_NAME = re.compile(r".*-ro-crate-metadata.json$")
 
 WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0"
 
@@ -95,9 +98,15 @@ def _has_writeable_stream(self):
         return True
 
     def write(self, dest_base):
-        write_path = Path(dest_base) / self.id
+        write_path = Path(dest_base) / self.id.rsplit("/", 1)[-1]
         super()._write_from_stream(write_path)
 
+    def write_detached(self, path):
+        if not DETACHED_MD_NAME.match(str(path)):
+            warnings.warn(f"{path} should follow the pattern {DETACHED_MD_NAME.pattern!r}")
+        path = Path(path)
+        super()._write_from_stream(path)
+
     @property
     def root(self) -> Dataset:
         return self.crate.root_dataset

rocrate/model/preview.py

@@ -28,6 +28,7 @@
 
 from jinja2 import Template
 from .file import File
+from ..utils import is_url
 
 
 class Preview(File):
@@ -99,6 +100,8 @@ def stream(self, chunk_size=8192):
             yield self.id, str.encode(self.generate_html(), encoding='utf-8')
 
     def _has_writeable_stream(self):
+        if is_url(str(self.source)):
+            return self.fetch_remote
         return True
 
     def write(self, dest_base):