Merge branch 'main' into dependabot/npm_and_yarn/anms-ui/public/axios-1.12.0

Author: BrianSipos

.env

@@ -1,3 +1,11 @@
+# This .env file is used by all compose commands for test and development usage.
+# NOTE: Production deployments via Puppet use an alternative version of this file generated by the tools.
+
+# Choose which profile(s) to run.
+#  If no profiles are set, a 'light' configuration will be started without the UI components
+#  Available profiles include 'full' (UI) and 'dev' (aeveloper tools such as adminer)
+COMPOSE_PROFILES=full,dev
+
 # Port Services; Uncomment below lines to override default mappings
 #AUTHNZ_PORT=8084
 #AUTHNZ_HTTPS_PORT=8443
@@ -7,7 +15,7 @@
 #MQTT_PORT=11883
 
 
-DOCKER_IMAGE_PREFIX=localhost/
+DOCKER_IMAGE_PREFIX=ghcr.io/nasa-ammos/anms/
 DOCKER_IMAGE_TAG=latest
 
 ANMS_VERSION=0

.github/workflows/anms-core.yaml

@@ -30,3 +30,34 @@ jobs:
         run: |
           FAIL_SRC=0
           flake8 src || FAIL_SRC=$?
+  anms-core_integration-test:
+    runs-on: ubuntu-24.04
+    env:
+      ANMS_COMPOSE_OPTS: -f docker-compose.yml --profile light
+      TEST_COMPOSE_OPTS: -f anms-core/integration_test/docker-compose.yml
+      HOST_SOCKDIR: /run/anms
+      DOCKER_CMD: docker
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Build ANMS
+        run: ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} build
+      - name: Build TEST
+        run: ${DOCKER_CMD} compose ${TEST_COMPOSE_OPTS} build
+      - name: Build Volume
+        run: |
+          ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls
+          sudo mkdir /run/anms
+      - name: Start
+        run: |
+          ${DOCKER_CMD} compose ${TEST_COMPOSE_OPTS} up -d test-transport
+          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} up -d --force-recreate --wait --wait-timeout 600
+      - name: Test
+        run: ${DOCKER_CMD} compose ${TEST_COMPOSE_OPTS} run test-runner
+      - name: Status
+        if: failure()
+        run: |
+          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} ps
+          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} logs anms-core

.github/workflows/build-test.yaml

@@ -60,6 +60,7 @@ jobs:
       - name: Status
         if: always()
         run: |
+          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} ps
           for BADSTATUS in stopped restarting; do
             ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} ps --services --filter status=${BADSTATUS} | tee -a /tmp/notgood
           done

.github/workflows/publish_images.yaml

@@ -0,0 +1,48 @@
+# On merge to main publish with the 'latest' label
+# Publish with release label when a releaes is published.
+name: Build and publish containers
+on:
+  push:
+    branches:
+    - main
+  release:
+    types: [published]
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set prefix
+        run: |
+          echo "DOCKER_IMAGE_PREFIX=${REGISTRY}/${REPO,,}/" >>${GITHUB_ENV}
+        env:
+          REPO: '${{ github.repository }}'
+      - name: Set image tag
+        id: vars
+        run: |
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            echo "DOCKER_IMAGE_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+          else
+            echo "DOCKER_IMAGE_TAG=latest" >> $GITHUB_ENV
+          fi
+
+      - name: Build and push base images via compose
+        run: |
+          docker compose build --push
+      - name: Build and push testenv images via compose
+        run: |
+          docker compose -f testenv-compose.yml build --push

.github/workflows/puppet.yaml

@@ -47,7 +47,7 @@ jobs:
           bundler-cache: true
 
       - name: Install puppet-lint
-        run: gem install puppet-lint
+        run: gem install puppet-lint -v 4.3.0
 
       - name: Run puppet-lint
         run: puppet-lint puppet --sarif --ignore-paths 'puppet/modules/anms/files/*' > puppet-lint-results.sarif

README.md

@@ -49,7 +49,16 @@ If your computer is behind a network proxy, this may cause issues related to usi
 Though ANMS can be run behind a proxy; building the ANMS Docker images from behind a network proxy may result in errors.
 
 The first steps in each of the container image `Dockerfile` is to attempt to download an APLNIS root CA to validate the APLNIS HTTPS proxy.
-When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.
+When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.  The URL for this certificate can be changed for users requiring equivalent functionality on their own networks.
+
+### Special Notes on Podman
+
+If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+
+Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+
+Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+
 
 ### Upgrading ANMS
 
@@ -63,32 +72,41 @@ The following command sequence uses standard Docker commands to stop all contain
 docker stop $(docker ps -q); docker rm $(docker ps --all -q); docker system prune -f; docker volume prune -f
 ```
 
-### Deployment Scenario
 
-The current ANMS capability is designed to run on `localhost` and on a development virtual machine.
-This guide presumes that you can either connect via a VMRC remote console or with ssh tunnelling to the machine, hence the use of `localhost` in db connection information and in URLs.
-If you deploy this to a VM, you will need to replace `localhost` with the hostname of the machine where it is deployed.
+## ANMS build and deploy
 
-### Special Notes on Podman
+## Quickstart
 
-If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+`./quickstart.sh`
 
-Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+The quickstart script will configure, pull, and start the ANMS system for the first time.  See comments in the script for additional details, including optional ENV variables to override default behavior.
 
-Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+NOTICE: By default, quick start will pull pre-built containers from the github registry (ghcr.io). To force a rebuild, run it as `FORCE_REBULD=y ./quickstart.sh`. See the script header for details.
 
+To stop the system use `podman compose -f testenv-compose.yml -f docker-compose.yml down`.
 
-## ANMS build and deploy
+To start the system in the future use `podman compose -f testenv-compose.yml up` and `podman compose up`.
 
+## Manual Startup
 Choose the appropriate docker, podman or podman-compose commands in the directions below as appropriate for your system.
 
-- Select appropriate profile(s) as desired. 
-  - If no profiles are set, a "light" deployment of the ANMS focused on browser-less API-only ANMS users.
-  - For a full deployment: `export COMPOSE_PROFILES=full`
-  - For a full deployment with additional developer tools: `export COMPOSE_PROFILES=full,dev`
+- Edit `.env` file as appropriately
+  - Select appropriate profile(s) as desired. 
+    - Core ANMS services are always started.
+    - The 'full' profile starts up all UI and related services.
+    - The 'dev' profile adds development tools, such as adminer
+    - Profiles can be set with COMPOSE_PROFILES in the .env file. The default includes full and dev profiles.
+  - Adjust network ports as necessary to avoid any conflicts or permissions issues.
+    - For rootless podman, the AUTHNZ_* ports must be changed to higher number ports to avoid permissions issues. 
+    - The corresponding lines can be uncommented in .env.
+- SELinux Security Labels Setup
+  - If your system does not support security labels, no additional steps are needed.
+  - If security labels are supported and you are unable to define them, they can be disabled for development purposes:
+    - `cp docker-compose.no-security-override.yml docker-compose.override.yml`
 - Clone this repository recursively (`git clone --recursive https://github.com/NASA-AMMOS/anms.git`)
 - Setup Volume containing PKI configuration (certificate chains and private keys):
   - `./create_volume.sh ./puppet/modules/apl_test/files/anms/tls`
+- OPTIONAL: The next 2 steps  will build all ANMS containers. If desired, these steps can be replaced with 'pull'ing prebuilt containers from ghcr.
 - Build Core Images using one of the following:
   - `docker compose -f docker-compose.yml build`
   - `podman compose -f docker-compose.yml build`
@@ -98,7 +116,6 @@ Choose the appropriate docker, podman or podman-compose commands in the directio
   - `docker compose -f testenv-compose.yml build`
   - `podman compose -f testenv-compose.yml build`
   - `podman-compose --podman-build-args='--format docker' -f testenv-compose.yml build`
-  
 - Start System using one of the following:
   - `docker compose -f docker-compose.yml up -d`
   - `podman compose -f docker-compose.yml up -d`
@@ -214,6 +231,8 @@ Refer to the `.env` file for port binding overrides, or `docker-compose.yml` for
 
 ### ANMS-UI is not visible at hostname
 
+Ensure that you are running with the 'full' profile. This is the default option when using the `.env` file, however some older versions of podman-compose may not parse the COMPOSE_PROFILES ENV variable correctly. If this is the case, specify the profile explicitly in your compose up commands. For example, `podman compose --profile full up`.
+
 Check the startup logs for any errors. If using podman, some port numbers may need to be remapped using the `.env` file to higher numbered ports, or the system configuration modified to adjust permissions (not recommended).
 
 If you go to your browser and hostname:9030 (replace hostname with the server's hostname) and you see the ANMS UI,

anms-core/anms/components/schemas/ARIs/registered_agent.py

@@ -31,7 +31,7 @@
 
 # Shared properties
 class RegisteredAgentBase(BaseModel):
-    agent_id_string: Optional[str] = None
+    agent_endpoint_uri: Optional[str] = None
     first_registered: Optional[datetime] = None
     last_registered: Optional[datetime] = None
 

anms-core/anms/components/schemas/ARIs/rpt_entry.py

@@ -29,11 +29,13 @@
 
 # Shared properties
 class RptEntryBase(BaseModel):
-    reference_time: Optional[str] = None
-    agent_id: Optional[str] = None
-    correlator_nonce: Optional[str] = None
+    class Config:
+        arbitrary_types_allowed = True
+
+    ari_rptset_id: Optional[str] = None
+    reference_time: Optional[datetime] = None
     report_list: Optional[str] = None
-    ari_rptset_id: Optional[int] = None
+    agent_id: Optional[int] = None
 
 
 class RptEntryBaseInDBBase(RptEntryBase):

anms-core/anms/models/relational/execution_set.py

@@ -35,7 +35,7 @@
 class ExecutionSet(Model):
     __tablename__ = 'vw_execution_set'
     execution_set_id = Column(Integer, primary_key=True)
-    correlator_nonce = Column(Integer)
+    nonce_cbor = Column(LargeBinary)
     use_desc = Column(String)
     agent_id = Column(String)
     num_entries = Column(Integer)
@@ -47,7 +47,7 @@ def __repr__(self) -> str:
     def as_dict(self) -> Dict[str, Any]:
         dict_obj = {
                     'execution_set_id': getattr(self, 'execution_set_id'),
-                    'correlator_nonce': getattr(self, 'correlator_nonce'),        
+                    'nonce_cbor': getattr(self, 'nonce_cbor'),        
                     'use_desc': getattr(self, 'use_desc'),
                     'agent_id': getattr(self, 'agent_id'),
                     'num_entries': getattr(self, 'num_entries'),

anms-core/anms/models/relational/registered_agent.py

@@ -37,7 +37,7 @@ class RegisteredAgent(Model):
     __tablename__ = 'registered_agents'
 
     registered_agents_id = Column(Integer, primary_key=True)
-    agent_id_string = Column(
+    agent_endpoint_uri = Column(
         String(128),
         default='ipn:0.0',
         unique=True,