ADDED: message of elapsed time, and added feature of downloading input data from shrepoint with argument to select if using local storage or sharepoint

Author: AleixMT

src/NAF.py

@@ -38,7 +38,7 @@ def slash_dash_str(self):
         return f"{self.province_code}/{self.middle_number}-{self.last_number}"
 
 
-def is_naf_correct(naf):
+def is_naf_format_correct(naf):
     """Validate that NAF has NAF format"""
     try:
         NAF(naf)  # Parse using constructor
@@ -47,13 +47,8 @@ def is_naf_correct(naf):
     return True
 
 
-def validate_parse_naf(value, valid_nafs):
-    if not is_naf_correct(value):
-        raise ArgumentNafInvalid("Naf " + value + " is not valid.")
-    elif NAF(value) not in valid_nafs:
-        raise ArgumentNafNotPresent("NAF value " + value + " is not in our NAF database (input/NAF_DNI.xlsx)")
-    else:
-        return NAF(value)
+def is_naf_present(value, valid_nafs):
+    return NAF(value) in valid_nafs
 
 
 def clean_naf(naf):
@@ -95,7 +90,4 @@ def parse_naf(naf):
     return dict(zip(naf_fixed, name_col))
 
 
-NAF_TO_DNI = build_naf_to_dni(NAF_DATA_PATH)
 
-# Build dictionaries to translate NAF to different identifier data
-NAF_TO_NAME = build_naf_to_name_and_surname(NAF_DATA_PATH)

src/arguments.py

@@ -3,7 +3,7 @@
 import sys
 from functools import partial
 
-from NAF import NAF_TO_DNI, validate_parse_naf
+from NAF import is_naf_present, NAF, build_naf_to_dni
 from custom_except import *
 from defines import NAF_DATA_PATH, DocType, from_string
 
@@ -12,6 +12,9 @@ def get_compact_init():
     return {DocType.SALARY: False, DocType.PROOFS: False, DocType.CONTRACT: False, DocType.RNT: False, DocType.RLC: False}
 
 
+# Parser functions that validate the format and type of the data
+
+
 def parse_date(value, formatting="%Y_%m"):
     """Validate date format"""
     try:
@@ -22,21 +25,17 @@ def parse_date(value, formatting="%Y_%m"):
 
 
 def parse_author(author):
-    with open("input/users.txt", newline="", encoding="utf-8") as f:
-        for line in f.readlines():
-            if line.__eq__(author):
-                return author
-        raise ArgumentAuthorError("Author " + author + " is not in the accepted user list (input/users.txt).")
+    return author
 
 
 def parse_naf(value):
     try:
-        return validate_parse_naf(value, NAF_TO_DNI.keys())
+        return NAF(value)
     except ValueError as e:
-        raise ArgumentNafNotPresent("NAF is not valid" + e.__str__())
+        raise ArgumentNafInvalid("NAF is not valid" + e.__str__())
 
 
-def parse_compact(value):
+def parse_compact_options(value):
     to_compact = get_compact_init()
     try:
         if "," in value:
@@ -52,7 +51,16 @@ def parse_compact(value):
         exit(1)
 
 
-def parse_arguments(valid_nafs):
+def parse_input_type(value):
+    if value == "sharepoint":
+        return value
+    elif value == "local":
+        return value
+    else:
+        raise UndefinedInputType("The type supplied for input type \"" + value + "\" is not defined.")
+
+
+def parse_arguments():
     """Parse and validate command-line arguments"""
     parser = argparse.ArgumentParser(description="Process NAF and date range.")
 
@@ -62,23 +70,27 @@ def parse_arguments(valid_nafs):
     parser.add_argument("-e", "--end", type=parse_date, required=True, help="End date (YYYY-MM)")
     parser.add_argument("-a", "--author", type=parse_author, required=True, help="author's email doing request")
 
-    parser.add_argument("-c", "--compact", type=parse_compact, required=False, default=get_compact_init(),
+    parser.add_argument("-c", "--compact", type=parse_compact_options, required=False, default=get_compact_init(),
                         help="Comma separated list of values that indicate which documents need to be merged in one "
                              "single PDF in the output. Possible values are: " +
                              ",".join([dt.value.__str__() for dt in DocType]))
 
+    parser.add_argument("-i", "--input", type=parse_input_type, required=False, default="sharepoint",
+                        help="Location of the input data. Possible values are: \"sharepoint\" to download from "
+                             "sharepoint location and \"local\" to use the local file system storage adn read the input"
+                             " folder in the repository root folder.")
     args = parser.parse_args()
 
     return args
 
 
-def process_arguments():
+def process_parse_arguments():
     common = ("Error parsing arguments. Program aborting. The arguments are: "
               + str(sys.argv) + "The program is in a uninitialized state and cannot proceed. This error will be "
                                 "notified to the admin via log file. We can't create log file in user author folder "
                                 "because user author could not be parsed.")
     try:
-        args = parse_arguments(NAF_TO_DNI.keys())
+        args = parse_arguments()
 
     except ArgumentNafNotPresent as e:
         print("The NAF provided is valid but is not present in " + NAF_DATA_PATH + ". Internal error is " + e.__str__())
@@ -103,3 +115,49 @@ def process_arguments():
     return args
 
 
+# Validations functions that check if the data from the request is valid regarding business rules
+
+
+def validate_naf(naf, valid_nafs):
+    if not is_naf_present(naf, valid_nafs):
+        raise ArgumentNafNotPresent
+
+
+def is_author_present(author, valid_authors):
+    return author in valid_authors
+
+
+def validate_author(author, valid_authors):
+    if not is_author_present(author, valid_authors):
+        raise ArgumentAuthorError("Author \"" + str(author) + " is not valid. ")  # more specific exception
+
+
+def validate_arguments(args, valid_nafs, valid_authors):
+    validate_author(args.author, valid_authors)
+    validate_naf(args.naf, valid_nafs)
+
+
+def process_validate_arguments(args, naf_data_path, user_list_data_path):
+    common = ("Error validating arguments. Program aborting. The arguments are: "
+              + str(sys.argv) + "The program is in a uninitialized state and cannot proceed. This error will be "
+                                "notified to the admin via log file. We can't create log file in user author folder "
+                                "because the process that validates user author could not finish.")
+
+    nafs = build_naf_to_dni(naf_data_path).keys()
+
+    authors = []
+    with open(user_list_data_path, newline="", encoding="utf-8") as f:
+        for line in f.readlines():
+            authors.append(line)
+
+    try:
+        validate_arguments(args, nafs, authors)
+
+    except ArgumentNafNotPresent as e:
+        print("The NAF provided is valid but is not present in " + naf_data_path + ". Internal error is " + e.__str__())
+        print(common)
+        exit(1)
+    except ArgumentAuthorError as e:
+        print("The author is not present in the accepted user list. Internal error is " + e.__str__())
+        print(common)
+        exit(4)

src/chrono.py

@@ -0,0 +1,26 @@
+import time
+from datetime import timedelta
+
+from logger import get_process_logger, get_logger_instance
+
+
+def elapsed_time(start_time):
+
+    elapsed = time.time() - start_time
+    delta = timedelta(seconds=int(elapsed))
+    days = delta.days
+    hours, remainder = divmod(delta.seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+
+    parts = []
+    if days:
+        parts.append(f"{days} day{'s' if days != 1 else ''}")
+    if hours:
+        parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
+    if minutes:
+        parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
+    if seconds or not parts:
+        parts.append(f"{seconds} second{'s' if seconds != 1 else ''}")
+
+    formatted = ', '.join(parts)
+    return formatted

src/custom_except.py

@@ -19,3 +19,6 @@ class UndefinedRegularSalaryType(Exception):
     pass
 
 
+class UndefinedInputType(Exception):
+    pass
+

src/defines.py

@@ -9,6 +9,8 @@
 
 # Obtain absolute path to the input directory
 INPUT_FOLDER = os.path.join(ROOT_FOLDER, "input")
+# Obtain absolute path to the valid user list
+USER_LIST_DATA_PATH = os.path.join(INPUT_FOLDER, "input")
 
 # Admin logs
 ADMIN_LOG_FOLDER = os.path.join(GENERAL_OUTPUT_FOLDER, "_admin_logs")
@@ -37,6 +39,7 @@ class DocType(Enum):
     RNT = "RNT"
     PROOFS = "proofs"
 
+
 def from_string(value: str):
     _aliases = {
         DocType.SALARY: {"salary", "salaries", "SALARY", "Salary", "payslip"},

src/filesystem.py

@@ -1,7 +1,7 @@
 import os
+import shutil
 from datetime import datetime
 
-from NAF import NAF_TO_NAME
 from defines import GENERAL_OUTPUT_FOLDER, ADMIN_LOG_FOLDER, SUPERVISOR_LOG_FOLDER, SALARIES_OUTPUT_NAME, \
     PROOFS_OUTPUT_NAME, CONTRACTS_OUTPUT_NAME, RNTS_OUTPUT_NAME, RLCS_OUTPUT_NAME
 
@@ -107,10 +107,10 @@ def flatten_dirs(folder_to_flat):
     return flatted_folders
 
 
-def compute_paths(args):
+def compute_paths(args, naf_to_name):
     NOW = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
 
-    id_str = (NOW + "_" + args.author + "_" + args.naf.__str__() + "_" + NAF_TO_NAME[args.naf] + "_" +
+    id_str = (NOW + "_" + args.author + "_" + args.naf.__str__() + "_" + naf_to_name[args.naf] + "_" +
               args.begin.strftime("%Y-%m") + "-" + args.end.strftime("%Y-%m") + ".log.txt")
     # Admin logs
     ADMIN_LOG_PATH = os.path.join(ADMIN_LOG_FOLDER, id_str)
@@ -120,18 +120,28 @@ def compute_paths(args):
     CURRENT_USER_FOLDER: str = os.path.join(GENERAL_OUTPUT_FOLDER, args.author)
 
     # Folder for the current justification
-    justification_name = (NOW + " " + str(args.naf) + " " + NAF_TO_NAME[args.naf] + " from " +
+    justification_name = (NOW + " " + str(args.naf) + " " + naf_to_name[args.naf] + " from " +
                           str(args.begin.strftime("%Y-%m")) + " to " +
                           str(args.end.strftime("%Y-%m")))
     CURRENT_JUSTIFICATION_FOLDER = os.path.join(CURRENT_USER_FOLDER, justification_name)
 
     USER_REPORT_FILE = os.path.join(CURRENT_JUSTIFICATION_FOLDER, NOW + "_" +
                                     args.naf.__str__()
-                                    + "_" + NAF_TO_NAME[args.naf].replace(" ", "_") + "_" +
+                                    + "_" + naf_to_name[args.naf].replace(" ", "_") + "_" +
                                     args.begin.strftime("%Y-%m") + "-" + args.end.strftime("%Y-%m") + ".log.txt")
     return CURRENT_USER_FOLDER, CURRENT_JUSTIFICATION_FOLDER, USER_REPORT_FILE, ADMIN_LOG_PATH, SUPERVISOR_LOG_PATH
 
 
+def remove_folder(folder_path):
+    """Remove the folder at the given path if it exists. Do nothing if it doesn't."""
+    try:
+        shutil.rmtree(folder_path)
+    except FileNotFoundError:
+        pass  # Do nothing if the folder does not exist
+    except Exception as e:
+        print(f"Error removing folder {folder_path}: {e}")
+
+
 def ensure_file_structure(CURRENT_USER_FOLDER, CURRENT_JUSTIFICATION_FOLDER):
     os.makedirs(GENERAL_OUTPUT_FOLDER, exist_ok=True)
     ensure_output_gitignore()

src/main.py

@@ -2,9 +2,10 @@
 import os.path
 import os.path
 import shutil
+import time
 
-from NAF import NAF, NAF_TO_DNI
-from arguments import parse_date, process_arguments
+from NAF import NAF, build_naf_to_dni, build_naf_to_name_and_surname
+from arguments import parse_date, process_parse_arguments
 from data import get_rlc_monthly_result_structure, parse_date_from_salary_filename, \
     parse_salary_filename_from_salary_path, unparse_date, parse_salary_type, unparse_month
 from defines import *
@@ -15,6 +16,8 @@
     merge_pdfs, compact_folder, parse_regular_salary_type, get_matching_pages
 from report import get_end_user_report, get_initial_user_report
 from custom_except import UndefinedRegularSalaryType
+from sharepoint import ensure_input_folder
+from chrono import elapsed_time
 
 
 def process_rlc_aux(salary_date, rlc_folder_path, months_found, rlc_subtype: str, rlc_type: str):
@@ -32,9 +35,9 @@ def process_rlc_aux(salary_date, rlc_folder_path, months_found, rlc_subtype: str
         return rlc_n_path
     else:
         logger.error("Monthly salary was found, but the expected L" + rlc_type + " RLC of type " + rlc_subtype + " was "
-                                                                                                                 "not found in the "
-                                                                                                                 "expected location "
-                                                                                                                 "" + str(
+                                                                                                     "not found in the "
+                                                                                                     "expected location "
+                                                                                                     "" + str(
             rlc_n_path) + ". Skipping merge of this salary file.")
         raise ValueError("File was not detected")  # TODO custom except
 
@@ -340,10 +343,22 @@ def process_RNTs(rnts_folder_path, naf_dir, naf, begin, end):
 
 
 def main():
-    args = process_arguments()
+    start_time = time.time()
+
+    args = process_parse_arguments()
+
+    # Ensure fresh input data
+    if args.input == "sharepoint":
+        ensure_input_folder(INPUT_FOLDER)
+    elif args.input == "local":
+        pass
+
+    start_time = time.time()
+
+    NAF_TO_DNI = build_naf_to_dni(NAF_DATA_PATH)
 
     current_user_folder, current_justification_folder, user_report_file, admin_log_path, supervisor_log_path = (
-        compute_paths(args))
+        compute_paths(args, NAF_TO_DNI))
 
     ensure_file_structure(current_user_folder, current_justification_folder)
 
@@ -357,6 +372,14 @@ def main():
 
     raw_logger.info(get_initial_user_report(args))
 
+    end_time = elapsed_time(start_time)
+    raw_logger.info("Time elapsed for downloading and validating input data: " + str(end_time) + ".")
+    start_time = time.time()
+
+    # Build dictionaries to translate NAF to different identifier data
+    NAF_TO_NAME = build_naf_to_name_and_surname(NAF_DATA_PATH)
+
+    # Begin processing
     reports = {}
     # Salaries & RLC
     reports[DocType.SALARY] = process_salaries_with_rlc(SALARIES_FOLDER, RLCS_FOLDER, current_justification_folder,
@@ -392,6 +415,12 @@ def main():
     report_text = get_end_user_report(reports, args)
     final_logger.info(report_text)
 
+    end_time = elapsed_time(start_time)
+    raw_logger.info("Time elapsed for doing this justification: " + str(end_time) + ".")
+    start_time = time.time()
+    elapsed_time(start_time)
+
+
 
 if __name__ == "__main__":
     main()