Feature/tag based ghcr images (#300)

* Grafana updates including: Upgraded to v12, switched internal DB to PostgresSQL, consolidated dashboard+widget definitions and moved to a text provisioning file, and updated anms-ui monitor to use the new dashboard.  Address #282 and #283

* Added to UI a prototype Help page and a Not Found page handler.

* Resolve default database warning in grafana (hopefully).

* Added upgrading file

* Add amp-manager to UI Status page

* added inifinity-datasource plugin and example panels

* updated layout of panels

* Fix core '/services' REST API to return JSON instead of text

* Core REST API bugfixes to detect timeouts and resolve potential error with overloaded status variable.

* initial dev creating a internal transcoder

* working version, added new profile default is internal transcoding

* working transcoder fixed issue with threading and DB

* Automatically create GHCR images when a tag is created. This supplements the formal release process which also triggers publication.

* Fix ghcr name if triggered by tag

* Fixing grafana authnz/demo websocket proxy configuration.

* removed services related to transcoder since they arent up by default

* added --ignore flag for podman volume creaete to avoid name error

* moved grafana db password to enviroment var

* Updated build-test workflow to use SHA instead of user-friendly branch in internal (not pushed) docker tags to prevent errors with docker not overwriting local tags by default.

* updated test to include new changes

---------

Co-authored-by: David Linko <david.linko@jhuapl.edu>

Author: DavidEdell

.env

@@ -7,8 +7,8 @@
 COMPOSE_PROFILES=full,dev
 
 # Port Services; Uncomment below lines to override default mappings
-#AUTHNZ_PORT=8084
-#AUTHNZ_HTTPS_PORT=8443
+AUTHNZ_PORT=8084
+AUTHNZ_HTTPS_PORT=8443
 #OPENSEARCH_PORT1=
 #OPENSEARCH_PORT2=
 #OPENSEARCH_DASH_PORT=
@@ -42,6 +42,8 @@ DB_HEALTHCHECK_PASSWORD=healthcheck
 GRAFANA_CONTAINER_PORT=3000
 GRAFANA_HOST_PORT=grafana:${GRAFANA_CONTAINER_PORT}
 GRAFANA_PROXIES_PATH=localhost/grafana
+GRAFANA_DB_NAME=grafana_internal_db
+
 REDIS_PORT=6379
 JS_AMP_PORT=3001
 ANMS_UI_HTTP_PORT=9030
@@ -55,7 +57,7 @@ KIBANA_PORT=5601
 ADMINER_PORT=8080
 RENDERER_PORT=8081
 RENDERER_HOST_PORT=grafana-image-renderer:${RENDERER_PORT}
-ION_MGR_PORT=8089
+ION_MGR_PORT=9089
 HTTP_PORT=80
 
 # Path (or volume name) on the host
@@ -64,3 +66,5 @@ HOST_SOCKDIR=sockdir
 CTR_SOCKDIR=/var/tmp/nm
 
 ADM_PATH=deps/dtnma-adms
+
+GRAFANA_PASS=grafana

.github/workflows/build-test.yaml

@@ -40,7 +40,7 @@ jobs:
           submodules: recursive
       - name: Tag name env
         run: |
-          DOCKER_IMAGE_TAG=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9\-\._]/-/g')
+          DOCKER_IMAGE_TAG=${{ github.sha }}
           echo "DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG}" >> $GITHUB_ENV
       - name: Build ANMS
         run: ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} build

.github/workflows/publish_images.yaml

@@ -5,6 +5,8 @@ on:
   push:
     branches:
     - main
+    tags:
+      - 'v*' # Triggers the workflow for any tag starting with 'v', e.g., v1.0.0
   release:
     types: [published]
 
@@ -36,6 +38,8 @@ jobs:
         run: |
           if [[ "${{ github.event_name }}" == "release" ]]; then
             echo "DOCKER_IMAGE_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+            echo "DOCKER_IMAGE_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
           else
             echo "DOCKER_IMAGE_TAG=latest" >> $GITHUB_ENV
           fi

UPGRADING.md

@@ -0,0 +1,17 @@
+This file will contain upgrading instructions for all future tagged releases.
+
+# Upgrading v2.0.0 to main
+
+## Grafana DB Update
+A new database named `grafana_internal_db` needs to be created in postgres. 
+
+This can be done from the commandline or via the dev UI. 
+
+In the latter case, ensure your instance is started with the `dev` and `full` profiles.  Go to Adminer (link provided in UI Help page) and either manually create the DB or 'Execute SQL' and upload the file `grafana/create_grafana_db.sql`
+
+# Upgrading v1.x to v.2.0.0
+
+It is recommended to start fresh (delete any existing ANMS-related containers and volumes) when transitioning from ANMS v1 to v2.
+
+If you have data or customizations in a v1 installation that you need to migrate, please contact us or open an issue to discuss.
+

anms-core/anms/config_template.yaml

@@ -108,4 +108,6 @@ EMAIL_ENABLED: EMAIL_ENABLED_TEMPLATE
 EMAIL_FROM_NAME: EMAIL_FROM_NAME_TEMPLATE 
 EMAIL_FROM_EMAIL: EMAIL_FROM_EMAIL_TEMPLATE 
 EMAIL_TEMPLATES_DIR: EMAIL_TEMPLATES_DIR_TEMPLATE 
-EMAIL_RESET_TOKEN_EXPIRE_LIFETIME: EMAIL_RESET_TOKEN_EXPIRE_LIFETIME_TEMPLATE 
+EMAIL_RESET_TOKEN_EXPIRE_LIFETIME: EMAIL_RESET_TOKEN_EXPIRE_LIFETIME_TEMPLATE 
+
+Transcoder: TRANSCODER_TEMPLATE

anms-core/anms/init_adms.py

@@ -31,6 +31,7 @@
 from anms.routes.adms.adm import handle_adm
 from anms.shared.config_utils import ConfigBuilder
 from anms.shared.opensearch_logger import OpenSearchLogger
+from anms.shared.transmogrifier import TRANSMORGIFIER
 
 
 logger = OpenSearchLogger(__name__).logger
@@ -67,17 +68,10 @@ async def import_adms():
             logger.error('ADM %s handling failed: %s', adm_file.norm_name, err)
             logger.debug('%s', traceback.format_exc())
 
+     
     # Notify the aricodec of startup
-    config = ConfigBuilder.get_config()
-    host = config.get('MQTT_HOST')
-    port = config.get('MQTT_PORT')
-
-    logger.info('Connecting to MQTT broker %s to notify aricodec', host)
-    client = mqtt.client.Client()
-    client.connect(host, port)
-    msg = client.publish('aricodec/reload', b'')
-    msg.wait_for_publish()
-    client.disconnect()
+    TRANSMORGIFIER.reload()
+    
 
     logger.info('Startup finished')
 

anms-core/anms/routes/adms/adm.py

@@ -29,20 +29,17 @@
 from pydantic import BaseModel
 import io
 import traceback
-from typing import TextIO
 
 # Internal modules
-from sqlalchemy import delete, select, and_
-from sqlalchemy.engine import Result
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import delete,  and_
 
 from anms.models.relational.adms import (adm_data, data_model_view)
-from anms.models.relational.adms.data_model_view import DataModel as ADM
 
 from anms.routes.adms.adm_compare import (AdmCompare)
 from anms.shared.opensearch_logger import OpenSearchLogger
-from anms.shared.mqtt_client import MQTT_CLIENT
-from anms.models.relational import get_async_session, get_session
+
+from anms.shared.transmogrifier import TRANSMORGIFIER
+from anms.models.relational import get_async_session
 from anms.components.schemas.adm import DataModelSchema
 import ace
 from camp.generators import (create_sql)
@@ -296,7 +293,7 @@ async def update_adm(file: UploadFile, request: Request):
                 if error_message:
                     raise Exception(error_message)
                 # Notify the transcoder
-                MQTT_CLIENT.publish('aricodec/reload', adm_file.norm_name)
+                TRANSMORGIFIER.reload(adm_file.norm_name)
                 logger.info(f"{info_message} adm file:  {file.filename} successfully")
             except Exception as err:
                 logger.error(f"{sql_dialect} execution error: {err.args}")

anms-core/anms/routes/network_manager.py

@@ -91,7 +91,17 @@ def do_nm_put_hex_eid(eid: str, ari: str):
     logger.info('post to nm manager %s  with eid %s and data %s' % (url, eid, ari))
 
     try:        
-        request = requests.post(url=url, data=ari, headers={'Content-Type': 'text/plain'})
+        request = requests.post(url=url,
+                                data=ari,
+                                headers={'Content-Type': 'text/plain'},
+                                timeout=(2.0, 8.0) # 2s for manager to connect, 8s for it to respond
+                                )
+    except requests.exceptions.ConnectTimeout:
+        return status.HTTP_504_GATEWAY_TIMEOUT
+    except requests.exceptions.ReadTimeout:
+        return status.HTTP_504_GATEWAY_TIMEOUT
+    except requests.exceptions.Timeout:
+        return status.HTTP_504_GATEWAY_TIMEOUT
     except Exception:
         return status.HTTP_500_INTERNAL_SERVER_ERROR
     return request.status_code

anms-core/anms/routes/system_status.py

@@ -138,4 +138,5 @@ async def sys_status_get_version():
 async def sys_status_get_services_status():
   statuses = get_containers_status()
   logger.debug(f"Checking all services' status: {str(statuses)}")
-  return json.dumps(statuses)
+  return statuses
+