187 improve initial default adm load process (#323)

* Grafana updates including: Upgraded to v12, switched internal DB to PostgresSQL, consolidated dashboard+widget definitions and moved to a text provisioning file, and updated anms-ui monitor to use the new dashboard.  Address #282 and #283

* Added to UI a prototype Help page and a Not Found page handler.

* Resolve default database warning in grafana (hopefully).

* Added upgrading file

* Add amp-manager to UI Status page

* added inifinity-datasource plugin and example panels

* updated layout of panels

* Fix core '/services' REST API to return JSON instead of text

* Core REST API bugfixes to detect timeouts and resolve potential error with overloaded status variable.

* initial dev creating a internal transcoder

* working version, added new profile default is internal transcoding

* working transcoder fixed issue with threading and DB

* Automatically create GHCR images when a tag is created. This supplements the formal release process which also triggers publication.

* Fix ghcr name if triggered by tag

* Fixing grafana authnz/demo websocket proxy configuration.

* removed services related to transcoder since they arent up by default

* added --ignore flag for podman volume creaete to avoid name error

* moved grafana db password to enviroment var

* moving adms init

* moved init adms logic into transcoder

* added new folder for adding new ADMS in core

* added gitignore to extra_adms to avoid extra adms be added to main

* moved init logic to avoid loop startup error

* loading default ADMs controlled by a new route

* removed uneeded f-string

---------

Co-authored-by: David Edell <david.edell@jhuapl.edu>
Co-authored-by: David Linko <david.linko@jhuapl.edu>

Author: 3 people

README.md

@@ -193,7 +193,7 @@ point. With ANMS running, go to `localhost:8080` and log in to the database with
 
 ### ADM and Agent Updates
 
-Changes to ADMs are handled on the Manager by uploading a new version of the ADM via the Web UI.
+By default after building the system, ANMS starts with the ADMs defined in `deps/dtnma-adms` and ADMs added to `anms-core/extra_adms`. Changes to and adding new ADMs are handled on the Manager by uploading a new version of the ADM via the Web UI or the REST POST endpoint `http://localhost:5555/adms/`.
 The manager will then be able to use the new ADM.
 
 Changes to a test Agent are more complicated, and require auto-generated C sources built into the ION source tree.

anms-core/anms/init_adms.py

@@ -22,7 +22,6 @@
 # subcontract 1658085.
 #
 import asyncio
-from functools import cache
 from typing import List
 
 from fastapi import Depends, APIRouter
@@ -36,7 +35,6 @@
 from anms.models.relational import get_async_session
 from anms.models.relational.actual_parameter import ActualParameter
 from anms.models.relational.ari import ARI
-from anms.models.relational.adms.data_model_view import DataModel 
 from anms.models.relational.formal_parameter import FormalParameter
 
 

anms-core/anms/routes/ARIs/ari.py

@@ -126,71 +126,11 @@ async def remove_adm(enumeration: int, namespace:str):
             logger.debug(f"ADM ENUM:{enumeration} in NAMESPACE {namespace} not a known ADM")
             raise HTTPException(status_code = status.HTTP_400_BAD_REQUEST, detail = f"ADM ENUM:{enumeration} in NAMESPACE {namespace} not a known ADM")
 
-
-
-async def handle_adm(admset: ace.AdmSet, adm_file: ace.models.AdmModule, session, replace=True):
-    ''' Process a received and decoded ADM into the ANMS DB.
-
-    :param replace: If true and the ADM exists it will be checked and replaced.
-    :return: A list of issues with the ADM, which is empty if successful.
-    '''
-    logger.info("Adm name: %s", adm_file.norm_name)
-    data_model_view = await DataModel.get(adm_file.ns_model_enum,adm_file.ns_org_name )
-    if data_model_view:
-        if not replace:
-            logger.info('Not replacing existing ADM name %s', adm_file.norm_name)
-            return []
-        data_rec = None
-        async with get_async_session() as session:
-            data_rec,_ = await AdmData.get(data_model_view.data_model_id,session)
-
-        if data_rec:
-            # Compare old and new contents
-            logger.info("Checking existing ADM name %s", adm_file.norm_name)
-            old_adm = admset.load_from_data(io.BytesIO(data_rec.data), del_dupe=False)
-            comp = AdmCompare(admset)
-            if not comp.compare_adms(old_adm, adm_file):
-                issues = comp.get_errors()
-            else:
-                issues = [f"Updating existing adm is not allowed yet"]
-            return issues
-
-    logger.info("Inserting ADM name %s", adm_file.norm_name)
-
-    # Use CAmPython to generate sql
-    out_path = ""  # This is empty string since we don't need to write the generated sql to a file
-    sql_dialect = 'pgsql'
-    writer = create_sql.Writer(admset, adm_file, out_path, sql_dialect)
-    string_buffer = io.StringIO()
-    writer.write(string_buffer)
-
-    # execute generated Sql
-    queries = string_buffer.getvalue()
-    try:
-        await session.execute(queries)
-        await session.commit()
-    except Exception as err:
-        logger.error(f"{sql_dialect} execution error: {err.args}")
-        logger.debug('%s', traceback.format_exc())
-        raise
-
-    # Save the adm file of the new adm
-    
-
-    buf = io.StringIO()
-    ace.adm_yang.Encoder().encode(adm_file, buf)
-    ret_dm = await DataModel.get(adm_file.ns_model_enum,  adm_file.ns_org_name, session)
-    
-    # Write the encoded string data to the BytesIO object
-    bytes_io = io.BytesIO()
-    bytes_io.write(buf.getvalue().encode('utf-8'))
-    # Reset the pointer to the beginning
-    bytes_io.seek(0)
-    data = {"enumeration":ret_dm.data_model_id, "data": bytes_io.getvalue()}
-    await AdmData.add_data(data, session)
-
-    return []
-
+@router.post("/load_default", status_code=status.HTTP_201_CREATED)
+async def load_default_adm():
+    await TRANSMORGIFIER.load_default_adms()
+    response = JSONResponse(status_code=status.HTTP_200_OK, content={"message": "Initilized default ADMs", "error_details": ""})
+    return response
 
 @router.post("/", status_code=status.HTTP_201_CREATED,
              responses={400: {"model": RequestError}, 405: {"model": UpdateAdmError}, 500: {"model": RequestError}})

anms-core/anms/routes/adms/adm.py

@@ -21,13 +21,17 @@
 # the prime contract 80NM0018D0004 between the Caltech and NASA under
 # subcontract 1658085.
 
-
+from camp.generators import (create_sql)
 from anms.shared.config import ConfigBuilder
-from anms.shared.logger import Logger
+import asyncio
 import anms.shared.mqtt_client
 from anms.shared.opensearch_logger import OpenSearchLogger
 from anms.models.relational import get_session
+from anms.models.relational import get_async_session
 from anms.models.relational.transcoder_log import TranscoderLog
+from anms.models.relational.adms import (adm_data, data_model_view)
+from anms.routes.adms.adm_compare import (AdmCompare)
+
 import traceback
 import ace
 import io
@@ -40,19 +44,20 @@
 
 
 # depending on what the config is for core will either use a MQTT server to send off commands or 
-# use an internal 
+# use an ACE internally to translate 
 class Transmorgifier:
 
     ''' The Transmogifier that can be configured to use an external or internal translator. '''
     # args = config 
     def __init__(self, args):
         # if the transcoding in internal to core
         LOGGER.info(config.Transcoder)
+        self.adm_data = adm_data.AdmData
+        self.data_model = data_model_view.DataModel
         if config.Transcoder == "Internal":
             db_uri = f"postgresql://{config.DB_USER}:{config.DB_PASS}@{config.DB_HOST}/{config.DB_CHROOT}"
             LOGGER.info(f'Connecting to SQL DB at {db_uri}')
             self._dbeng = sqlalchemy.create_engine(db_uri)
-            
             self.transcode = self._transcode_internal
             self.reload = self._reload_internal
             self._adm_reload(None)
@@ -61,6 +66,87 @@ def __init__(self, args):
             self.MQTT_CLIENT = anms.shared.mqtt_client.MQTT_CLIENT
             self.transcode = self._transcode_mqtt
             self.reload = self._reload_mqtt 
+    
+    async def handle_adm(self, admset: ace.AdmSet, adm_file: ace.models.AdmModule, session, replace=True):
+        ''' Process a received and decoded ADM into the ANMS DB.
+
+        :param replace: If true and the ADM exists it will be checked and replaced.
+        :return: A list of issues with the ADM, which is empty if successful.
+        '''
+        LOGGER.info("Adm name: %s", adm_file.norm_name)
+        data_model_view = await self.data_model.get(adm_file.ns_model_enum,adm_file.ns_org_name )
+        if data_model_view:
+            if not replace:
+                LOGGER.info('Not replacing existing ADM name %s', adm_file.norm_name)
+                return []
+            data_rec = None
+            async with get_async_session() as session:
+                data_rec,_ = await self.adm_data.get(data_model_view.data_model_id,session)
+
+            if data_rec:
+                # Compare old and new contents
+                LOGGER.info("Checking existing ADM name %s", adm_file.norm_name)
+                old_adm = admset.load_from_data(io.BytesIO(data_rec.data), del_dupe=False)
+                comp = AdmCompare(admset)
+                if not comp.compare_adms(old_adm, adm_file):
+                    issues = comp.get_errors()
+                else:
+                    issues = ["Updating existing adm is not allowed yet"]
+                return issues
+
+        LOGGER.info("Inserting ADM name %s", adm_file.norm_name)
+
+        # Use CAmPython to generate sql
+        out_path = ""  # This is empty string since we don't need to write the generated sql to a file
+        sql_dialect = 'pgsql'
+        writer = create_sql.Writer(admset, adm_file, out_path, sql_dialect)
+        string_buffer = io.StringIO()
+        writer.write(string_buffer)
+
+        # execute generated Sql
+        queries = string_buffer.getvalue()
+        try:
+            await session.execute(queries)
+            await session.commit()
+        except Exception as err:
+            LOGGER.error(f"{sql_dialect} execution error: {err.args}")
+            LOGGER.debug('%s', traceback.format_exc())
+            raise
+
+        # Save the adm file of the new adm
+        buf = io.StringIO()
+        ace.adm_yang.Encoder().encode(adm_file, buf)
+        ret_dm = await self.data_model.get(adm_file.ns_model_enum,  adm_file.ns_org_name, session)
+        
+        # Write the encoded string data to the BytesIO object
+        bytes_io = io.BytesIO()
+        bytes_io.write(buf.getvalue().encode('utf-8'))
+        # Reset the pointer to the beginning
+        bytes_io.seek(0)
+        data = {"enumeration":ret_dm.data_model_id, "data": bytes_io.getvalue()}
+        await self.adm_data.add_data(data, session)
+
+        return []
+            
+    async def load_default_adms(self):
+        admset = ace.AdmSet(cache_dir=False)
+        admset.load_default_dirs()
+        issues = ace.Checker(admset.db_session()).check()
+        for iss in issues:
+            LOGGER.error('ADM issue %s', iss)
+
+        for adm_file in admset:
+            try:
+                LOGGER.info('ADM %s handling started', adm_file.norm_name)
+                async with get_async_session() as db_sess:
+                    await self.handle_adm(admset, adm_file, db_sess, replace=False)
+                LOGGER.info('ADM %s handling finished', adm_file.norm_name)
+            except Exception as err:
+                # The function already logged any SQL issue at error severity
+                LOGGER.error('ADM %s handling failed: %s', adm_file.norm_name, err)
+                LOGGER.debug('%s', traceback.format_exc())
+
+        self.reload()
 
     def _transcode_mqtt(self, input):
         msg = json.dumps({'uri': input})
@@ -101,7 +187,6 @@ def _ace_transcode(self, input):
                     dec = ace.ari_cbor.Decoder()
                     ari = dec.decode(io.BytesIO(in_bytes))
                     LOGGER.debug(f'decoded as ARI {ari}')
-                    # ace.nickname.Converter(ace.nickname.Mode.FROM_NN, admsSession(self._dbeng), True)(ari)
                     ari = ace.nickname.Converter(ace.nickname.Mode.FROM_NN, adms.db_session(), False)(ari)
 
                 except Exception as err:

anms-core/anms/shared/transmogrifier.py

@@ -27,7 +27,4 @@ set -e
 
 PYTHON=${PYTHON:-python3}
 
-# initialize DB state
-${PYTHON} -m anms.init_adms
-
 exec ${PYTHON} -m anms.run_gunicorn

anms-core/docker-entrypoint.sh

@@ -0,0 +1,3 @@
+#ignoring any added ADMs to this file to avoid uploading test or unwanted ADMs  
+*
+!.gitignore

anms-core/extra_adms/.gitignore

@@ -108,13 +108,16 @@ export default {
   },
   async mounted() {
     if (!this.hasAdms) {
+      await this.loadAdms();
       await this.getAdms();
     }
   },
   methods: {
     ...mapActions("adm", {
       getAdms: "getAdms",
-      uploadAdm: "uploadAdm"
+      uploadAdm: "uploadAdm",
+      loadAdms: "loadAdms"
+
     }),
     download(adm){
       let json  = {};

anms-ui/public/app/components/adm/Adm.vue

@@ -53,8 +53,13 @@ const apiUpdateAdm = async (file) => {
     );
 };
 
+const apiLoadAdm = async () => {
+    return axios.post(adm_url+"/load_default", {headers: {accept: 'application/json'}})
+};
+
 export default {
     apiGetAdms,
     apiGetAdm,
-    apiUpdateAdm
+    apiUpdateAdm,
+    apiLoadAdm
 };

anms-ui/public/app/shared/api_adm.js

@@ -72,6 +72,19 @@ export default {
 
       });
     },
+    async loadAdms({ state, commit }){
+      commit('loading', true);
+      commit('requestError', "");
+      let sleep = (time) => new Promise((resolve) => setTimeout(resolve, time));
+      api_adm.apiLoadAdm()
+      .catch(function (error) {
+        sleep(1000).then(() => {
+          commit('adms', []);
+          commit('requestError', error);
+          commit('loading', false);
+        })
+      });
+    },
     async uploadAdm({ state, commit }, adm_file){
       //commit('loading', true);
       commit('requestError', "");