Enforce consistent encodings throughout when loading templates (#547)

Author: caufieldjh

src/ontogpt/cli.py

@@ -55,6 +55,7 @@
 from ontogpt.io.owl_exporter import OWLExporter
 from ontogpt.io.rdf_exporter import RDFExporter
 from ontogpt.io.template_loader import get_template_details, get_template_path
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.io.yaml_wrapper import dump_minimal_yaml
 from ontogpt.templates.core import ExtractionResult
 from ontogpt.utils.multilingual import multilingual_analysis
@@ -196,7 +197,7 @@ def parse_input(
                         parse_tabular_input(str(f), selectcols)
                         if Path(f).suffix in VALID_TABULAR_FORMATS
                         or Path(f).suffix in VALID_SPREADSHEET_FORMATS
-                        else open(f, "r").read()
+                        else read_text_with_fallbacks(f)
                     )
                     for f in inputfiles
                     if f.is_file()
@@ -207,7 +208,7 @@ def parse_input(
                         parse_tabular_input(str(f), selectcols)
                         if Path(f).suffix in VALID_TABULAR_FORMATS
                         or Path(f).suffix in VALID_SPREADSHEET_FORMATS
-                        else open(f, "r").read()
+                        else read_text_with_fallbacks(f)
                     )
                     for f in inputfiles
                     if f.is_file()
@@ -225,7 +226,7 @@ def parse_input(
             text = parse_tabular_input(input, selectcols)
             logging.info(f"Input text: {text}")
         else:
-            text = open(input, "rb").read().decode(encoding="utf-8", errors="ignore")
+            text = read_text_with_fallbacks(Path(input))
             logging.info(f"Input text: {text}")
         parsedlist = [text]
     else:
@@ -1102,11 +1103,11 @@ def recipe_extract(
         ke.client.cache_db_path = settings.cache_db
 
     if recipes_urls_file:
-        with open(recipes_urls_file, "r") as f:
-            urls = [line.strip() for line in f.readlines() if url in line]
-            if len(urls) != 1:
-                raise ValueError(f"Found {len(urls)} URLs in {recipes_urls_file}")
-            url = urls[0]
+        content = read_text_with_fallbacks(Path(recipes_urls_file))
+        urls = [line.strip() for line in content.splitlines() if url in line]
+        if len(urls) != 1:
+            raise ValueError(f"Found {len(urls)} URLs in {recipes_urls_file}")
+        url = urls[0]
     scraper = scrape_me(url)
 
     if dictionary:
@@ -1172,8 +1173,7 @@ def convert(
     )
 
     cls = ke.template_pyclass
-    with open(input, "r") as f:
-        data = yaml.safe_load(f)
+    data = yaml.safe_load(read_text_with_fallbacks(Path(input)))
     obj = cls(**data["extracted_object"])
     results = ExtractionResult(extracted_object=obj)
     write_extraction(results, output, output_format, ke, template, cut_input_text)
@@ -1580,7 +1580,7 @@ def diagnose(
     if not model:
         model = DEFAULT_MODEL
 
-    phenopackets = [json.load(open(f)) for f in phenopacket_files]
+    phenopackets = [json.loads(read_text_with_fallbacks(Path(f))) for f in phenopacket_files]
     engine = PhenoEngine(
         model=model,
         temperature=temperature,
@@ -1633,7 +1633,9 @@ def run_multilingual_analysis(
     elif input_data_dir and Path(input_data_dir).is_dir():
         logging.info(f"Input file directory: {input_data_dir}")
         inputfiles = Path(input_data_dir).glob("*.txt")
-        inputdict = {str(f.name): open(f, "r").read() for f in inputfiles if f.is_file()}
+        inputdict = {
+            str(f.name): read_text_with_fallbacks(f) for f in inputfiles if f.is_file()
+        }
         logging.info(f"Found {len(inputdict)} input files here.")
 
     i = 0
@@ -1698,7 +1700,7 @@ def answer(
     **kwargs,
 ):
     """Answer a set of questions defined in YAML."""
-    qc = QuestionCollection(**yaml.safe_load(open(inputfile)))
+    qc = QuestionCollection(**yaml.safe_load(read_text_with_fallbacks(Path(inputfile))))
     engine = GenericEngine(
         model=model,
         temperature=temperature,
@@ -1947,7 +1949,7 @@ def complete(
     """
 
     if inputfile:
-        text = open(inputfile).read()
+        text = read_text_with_fallbacks(Path(inputfile))
     else:
         text = input.strip()
 
@@ -2174,7 +2176,7 @@ def _get_templates() -> Dict[str, Tuple[str, str]]:
     template_dir = Path(__file__).parent / "templates"
     template_paths = [f for f in template_dir.glob("*.yaml")]
     for template_path in template_paths:
-        with open(template_path, "r") as template_file:
+        with open(template_path, "r", encoding="utf-8") as template_file:
             data = yaml.safe_load(template_file)
             if data["id"].startswith(http_prefixes):
                 identifier = data["id"].split("/")[-1]

src/ontogpt/engines/generic_engine.py

@@ -35,6 +35,7 @@
 from pydantic import BaseModel
 
 from ontogpt.engines.knowledge_engine import KnowledgeEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 
 logger = logging.getLogger(__name__)
 
@@ -77,9 +78,8 @@ def run(
         if isinstance(template_path, str):
             # Note: The default is a string, so this will always be true
             # create a Jinja2 template object
-            with open(template_path) as file:
-                template_txt = file.read()
-                main_template = Template(template_txt)
+            template_txt = read_text_with_fallbacks(Path(template_path))
+            main_template = Template(template_txt)
         for question in question_collection.questions:
             for instruction in question_collection.instructions:
                 template = main_template

src/ontogpt/engines/halo_engine.py

@@ -23,6 +23,7 @@
 
 from ontogpt.clients import LLMClient
 from ontogpt.engines.knowledge_engine import FIELD, KnowledgeEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.io.yaml_wrapper import dump_minimal_yaml
 from ontogpt.templates.halo import Ontology, OntologyElement
 
@@ -96,7 +97,8 @@ def seed_from_file(self, file_path: str) -> Ontology:
         :param file_path:
         :return:
         """
-        ontology = Ontology(**yaml.safe_load(open(file_path)))
+        ontology_text = read_text_with_fallbacks(Path(file_path))
+        ontology = Ontology(**yaml.safe_load(ontology_text))
         self.seed(ontology)
         logger.info(f"Seeded with {len(ontology.elements)} elements")
         return ontology

src/ontogpt/engines/knowledge_engine.py

@@ -94,6 +94,7 @@
 from ontogpt import DEFAULT_MODEL
 from ontogpt.clients.llm_client import LLMClient
 from ontogpt.templates.core import ExtractionResult, NamedEntity
+from ontogpt.io.utils import read_text_with_fallbacks
 
 this_path = Path(__file__).parent
 logger = logging.getLogger(__name__)
@@ -271,8 +272,7 @@ def extract_from_file(self, file: Union[str, Path, TextIO]) -> pydantic.BaseMode
         if isinstance(file, str):
             file = Path(file)
         if isinstance(file, Path):
-            with file.open() as f:
-                text = f.read()
+            text = read_text_with_fallbacks(file)
         else:
             text = file.read()
         self.last_text = text
@@ -283,8 +283,8 @@ def extract_from_file(self, file: Union[str, Path, TextIO]) -> pydantic.BaseMode
     def load_dictionary(self, path: Union[str, Path, list]):
         if not isinstance(path, list):
             logger.info(f"Loading dictionary from {path}")
-            with open(str(path)) as f:
-                return self.load_dictionary(yaml.safe_load(f))
+            data = read_text_with_fallbacks(Path(path))
+            return self.load_dictionary(yaml.safe_load(data))
         if self.dictionary is None:
             self.dictionary = {}
         entries = [(entry["synonym"].lower(), entry["id"]) for entry in path]

src/ontogpt/engines/mapping_engine.py

@@ -47,6 +47,7 @@
 from sssom_schema import Mapping
 
 from ontogpt.engines.knowledge_engine import KnowledgeEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.prompts.mapping import DEFAULT_MAPPING_EVAL_PROMPT
 
 logger = logging.getLogger(__name__)
@@ -153,9 +154,8 @@ def categorize_mapping(
             template_path = str(template_path)
         if isinstance(template_path, str):
             # create a Jinja2 template object
-            with open(template_path) as file:
-                template_txt = file.read()
-                template = Template(template_txt)
+            template_txt = read_text_with_fallbacks(Path(template_path))
+            template = Template(template_txt)
         subject_concept = self._concept(subject, self.subject_adapter)
         object_concept = self._concept(object, self.object_adapter)
         prompt = template.render(

src/ontogpt/engines/pheno_engine.py

@@ -13,6 +13,7 @@
 from pydantic import BaseModel
 
 from ontogpt.engines.knowledge_engine import KnowledgeEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.io.yaml_wrapper import dump_minimal_yaml
 from ontogpt.prompts.phenopacket import DEFAULT_PHENOPACKET_PROMPT
 
@@ -58,9 +59,8 @@ def predict_disease(
             template_path = str(template_path)
         if isinstance(template_path, str):
             # create a Jinja2 template object
-            with open(template_path) as file:
-                template_txt = file.read()
-                template = Template(template_txt)
+            template_txt = read_text_with_fallbacks(Path(template_path))
+            template = Template(template_txt)
         # Account for missing template fields if necessary
         # TODO: make this its own function
         for subject_key in ["sex", "ageAtCollection"]:

src/ontogpt/engines/reasoner_engine.py

@@ -37,6 +37,7 @@
 from pydantic import BaseModel
 
 from ontogpt.engines.knowledge_engine import KnowledgeEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.ontex.extractor import (
     Answer,
     Axiom,
@@ -177,9 +178,8 @@ def reason(
             template_path = str(template_path)
         if isinstance(template_path, str):
             # create a Jinja2 template object
-            with open(template_path) as file:
-                template_txt = file.read()
-                template = Template(template_txt)
+            template_txt = read_text_with_fallbacks(Path(template_path))
+            template = Template(template_txt)
         prompt = template.render(
             task=task,
             ontology=task.ontology,

src/ontogpt/engines/spires_engine.py

@@ -32,6 +32,7 @@
     chunk_text_by_char,
     chunk_text_by_sentence,
 )
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.io.yaml_wrapper import dump_minimal_yaml
 from ontogpt.templates.core import ExtractionResult, NamedEntity
 from ontogpt.utils.parse_utils import get_span_values, sanitize_text
@@ -212,7 +213,7 @@ def _remove_parenthetical_context(s: str):
             cache_path = Path(cache_path)
         if cache_path:
             if cache_path.exists() and not clear:
-                db = yaml.safe_load(cache_path.open())
+                db = yaml.safe_load(read_text_with_fallbacks(cache_path))
                 if "entities_in_queue" not in db:
                     db["entities_in_queue"] = []
             else:

src/ontogpt/evaluation/drugmechdb/eval_drugmechdb.py

@@ -20,6 +20,7 @@
 import ontogpt.templates.drug as target_datamodel
 from ontogpt.engines.spires_engine import SPIRESEngine
 from ontogpt.evaluation.evaluation_engine import SimilarityScore, SPIRESEvaluationEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 
 THIS_DIR = Path(__file__).parent
 DATABASE_DIR = Path(__file__).parent / "database"
@@ -115,16 +116,15 @@ def drug_to_mechanism_text(self) -> Dict[str, str]:
         :return:
         """
         if self._drug_to_mechanism_text is None:
-            with open(MOA_TEXTS, "r") as f:
-                self._drug_to_mechanism_text = {}
-                for line in f:
-                    toks = line.strip().split("\t")
-                    if len(toks) != 2:
-                        logging.warning(f"Bad line: {line}")
-                        continue
-                    drug, text = toks
-                    if text != "NA":
-                        self._drug_to_mechanism_text[f"drugbank:{drug}"] = text
+            self._drug_to_mechanism_text = {}
+            for line in read_text_with_fallbacks(Path(MOA_TEXTS)).splitlines():
+                toks = line.strip().split("\t")
+                if len(toks) != 2:
+                    logging.warning(f"Bad line: {line}")
+                    continue
+                drug, text = toks
+                if text != "NA":
+                    self._drug_to_mechanism_text[f"drugbank:{drug}"] = text
         return self._drug_to_mechanism_text
 
     def load_source_mechanisms_from_path(
@@ -133,9 +133,10 @@ def load_source_mechanisms_from_path(
         if isinstance(file, Path):
             file = str(file)
         if isinstance(file, str):
-            with open(file, "r") as f:
-                return self.load_source_mechanisms_from_path(f)
-        mechanisms = yaml.safe_load(file)
+            data = read_text_with_fallbacks(Path(file))
+            mechanisms = yaml.safe_load(data)
+        else:
+            mechanisms = yaml.safe_load(file)
         print(f"Loading {len(mechanisms)} mechanism objects from yaml; translating...")
         return [source_datamodel.Mechanism(**_fix_source_mechanism(m)) for m in mechanisms]
 

src/ontogpt/evaluation/go/eval_go.py

@@ -13,6 +13,7 @@
 
 from ontogpt.engines.spires_engine import SPIRESEngine
 from ontogpt.evaluation.evaluation_engine import SimilarityScore, SPIRESEvaluationEngine
+from ontogpt.io.utils import read_text_with_fallbacks
 from ontogpt.templates.metabolic_process import MetabolicProcess
 
 TEST_CASES_DIR = Path(__file__).parent / "test_cases"
@@ -82,8 +83,8 @@ def make_term_from_ldef(self, ldef: LogicalDefinitionAxiom) -> MetabolicProcess:
         return mp
 
     def valid_test_ids(self) -> List[str]:
-        with open(TEST_CASES_DIR / "go-ids-2022.txt") as f:
-            return [x.strip() for x in f.readlines()]
+        data = read_text_with_fallbacks(TEST_CASES_DIR / "go-ids-2022.txt")
+        return [x.strip() for x in data.splitlines()]
 
     def ldef_matches(self, ldef: LogicalDefinitionAxiom) -> bool:
         """Check if a logical definition matches the genus and differentia."""