Merge branch 'main' into 235-uploading-new-yang-file-is-broken

Author: David Linko

anms-ui/.dockerignore .dockerignoreanms-ui/.dockerignore renamed to .dockerignore

@@ -1,27 +1,27 @@
 # SCM files
-.git
+.git/
 .gitignore
 
 # Ignore IDE/IntelliJ Files
 .idea
 *.iml
 
-# Ignore Node Modules used locally
-public/node_modules
-server/node_modules
-
+# Ignore build files
+node_modules
+/puppet/.modules/
 # Ignore Compiled Vue App
-release
-
+/anms-ui/release
 # Ignore Data Directory
-logs
-
+/anms-ui/logs
 
 # Ignore Other Specific Stuff
-docker-compose.yaml
+*-compose.yaml
+Containerfile
 Dockerfile
 Makefile
 .dockerignore
 gl-sast-report.json
 .metrics-dashboard.yml
-CONTRIBUTING.md
+*.md
+*.pdf
+*.docx

.env

@@ -1,3 +1,11 @@
+# This .env file is used by all compose commands for test and development usage.
+# NOTE: Production deployments via Puppet use an alternative version of this file generated by the tools.
+
+# Choose which profile(s) to run.
+#  If no profiles are set, a 'light' configuration will be started without the UI components
+#  Available profiles include 'full' (UI) and 'dev' (aeveloper tools such as adminer)
+COMPOSE_PROFILES=full,dev
+
 # Port Services; Uncomment below lines to override default mappings
 #AUTHNZ_PORT=8084
 #AUTHNZ_HTTPS_PORT=8443
@@ -7,8 +15,6 @@
 #MQTT_PORT=11883
 
 
-
-DOCKER_CTR_PREFIX=
 DOCKER_IMAGE_PREFIX=localhost/
 DOCKER_IMAGE_TAG=latest
 
@@ -34,8 +40,8 @@ DB_HEALTHCHECK_USER=healthcheck
 DB_HEALTHCHECK_PASSWORD=healthcheck
 
 GRAFANA_CONTAINER_PORT=3000
-GRAFANA_HOST_PORT=${DOCKER_CTR_PREFIX}grafana:${GRAFANA_CONTAINER_PORT}
-GRAFANA_PROXIES_PATH=localhost/${DOCKER_CTR_PREFIX}grafana
+GRAFANA_HOST_PORT=grafana:${GRAFANA_CONTAINER_PORT}
+GRAFANA_PROXIES_PATH=localhost/grafana
 REDIS_PORT=6379
 JS_AMP_PORT=3001
 ANMS_UI_HTTP_PORT=9030
@@ -48,7 +54,7 @@ LOGSTASH_MONITORING_PORT=9600
 KIBANA_PORT=5601
 ADMINER_PORT=8080
 RENDERER_PORT=8081
-RENDERER_HOST_PORT=${DOCKER_CTR_PREFIX}grafana-image-renderer:${RENDERER_PORT}
+RENDERER_HOST_PORT=grafana-image-renderer:${RENDERER_PORT}
 ION_MGR_PORT=8089
 HTTP_PORT=80
 

.github/workflows/build-test.yaml

@@ -17,7 +17,8 @@ jobs:
     name: Checkout Test (${{matrix.ctrmgr}})
     env:
       AUTHNZ_EMU: "demo"
-      ANMS_COMPOSE_OPTS: "-f docker-compose.yml --profile=full"
+      COMPOSE_PROFILES: "full"
+      ANMS_COMPOSE_OPTS: "-f docker-compose.yml"
       TESTENV_COMPOSE_OPTS: "-f testenv-compose.yml"
       DOCKER_CMD: ${{matrix.ctrmgr}}
       AUTHNZ_PORT: 8084
@@ -42,9 +43,7 @@ jobs:
           DOCKER_IMAGE_TAG=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9\-\._]/-/g')
           echo "DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG}" >> $GITHUB_ENV
       - name: Build ANMS
-        run: |
-          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} build builder-base builder-init builder-acelib
-          ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} build
+        run: ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} build
       - name: Build Agents
         run: ${DOCKER_CMD} compose ${TESTENV_COMPOSE_OPTS} build
       - name: Build Volume
@@ -59,6 +58,7 @@ jobs:
           ${DOCKER_CMD} compose ${TESTENV_COMPOSE_OPTS} up -d --force-recreate
           ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} up -d --force-recreate --wait --wait-timeout 600
       - name: Status
+        if: always()
         run: |
           for BADSTATUS in stopped restarting; do
             ${DOCKER_CMD} compose ${ANMS_COMPOSE_OPTS} ps --services --filter status=${BADSTATUS} | tee -a /tmp/notgood
@@ -70,14 +70,7 @@ jobs:
           # Fail if any names are in the file
           ! grep '[^[:space:]]' /tmp/notgood
       - name: Test
-        run: |
-          # Checkout the running gateway+backend
-          ${DOCKER_CMD} build -t checkout-test checkout-test
-          ${DOCKER_CMD} run --network anms -v $PWD:/mnt \
-              -e XUNIT_OUTFILE=/mnt/testresults.xml \
-              -e CHECKOUT_BASE_URL=http://authnz/ \
-              -e SSL_CERT_FILE=/mnt/puppet/modules/apl_test/files/anms/tls/certs/ammos-ca-bundle.crt \
-              checkout-test
+        run: ./checkout-test/run.sh
       - name: Stop
         if: always()
         run: |

.gitignore

@@ -7,6 +7,8 @@
 .DS_Store
 .project
 .cproject
+.pydevproject
+.settings/
 
 # Python intermediates
 __pycache__
@@ -16,11 +18,9 @@ coverage.xml
 testresults.xml
 
 # local build files
-anms-core/build/
-anms-ui/release/
+/anms-core/build/
+/anms-ui/release/
 
 # Javascript and ESLint related content to be ignored
 node_modules
 package-lock.json
-
-ion/configs/**/logs

README.md

@@ -49,7 +49,16 @@ If your computer is behind a network proxy, this may cause issues related to usi
 Though ANMS can be run behind a proxy; building the ANMS Docker images from behind a network proxy may result in errors.
 
 The first steps in each of the container image `Dockerfile` is to attempt to download an APLNIS root CA to validate the APLNIS HTTPS proxy.
-When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.
+When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.  The URL for this certificate can be changed for users requiring equivalent functionality on their own networks.
+
+### Special Notes on Podman
+
+If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+
+Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+
+Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+
 
 ### Upgrading ANMS
 
@@ -63,29 +72,35 @@ The following command sequence uses standard Docker commands to stop all contain
 docker stop $(docker ps -q); docker rm $(docker ps --all -q); docker system prune -f; docker volume prune -f
 ```
 
-### Deployment Scenario
 
-The current ANMS capability is designed to run on `localhost` and on a development virtual machine.
-This guide presumes that you can either connect via a VMRC remote console or with ssh tunnelling to the machine, hence the use of `localhost` in db connection information and in URLs.
-If you deploy this to a VM, you will need to replace `localhost` with the hostname of the machine where it is deployed.
-
-### Special Notes on Podman
+## ANMS build and deploy
 
-If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+## Quickstart
 
-Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+`./quickstart.sh`
 
-Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+The quickstart script will configure, build, and start the ANMS system for the first time.  See comments in the script for additional details, including optional ENV variables to override default behavior.
 
+To stop the system use `podman compose -f testenv-compose.yml -f docker-compose.yml down`.
 
-## ANMS build and deploy
+To start the system in the future use `podman compose -f testenv-compose.yml up` and `podman compose up`.
 
+## Manual Startup
 Choose the appropriate docker, podman or podman-compose commands in the directions below as appropriate for your system.
 
-- Select appropriate profile(s) as desired. 
-  - If no profiles are set, a "light" deployment of the ANMS focused on browser-less API-only ANMS users.
-  - For a full deployment: `export COMPOSE_PROFILES=full`
-  - For a full deployment with additional developer tools: `export COMPOSE_PROFILES=full,dev`
+- Edit `.env` file as appropriately
+  - Select appropriate profile(s) as desired. 
+    - Core ANMS services are always started.
+    - The 'full' profile starts up all UI and related services.
+    - The 'dev' profile adds development tools, such as adminer
+    - Profiles can be set with COMPOSE_PROFILES in the .env file. The default includes full and dev profiles.
+  - Adjust network ports as necessary to avoid any conflicts or permissions issues.
+    - For rootless podman, the AUTHNZ_* ports must be changed to higher number ports to avoid permissions issues. 
+    - The corresponding lines can be uncommented in .env.
+- SELinux Security Labels Setup
+  - If your system does not support security labels, no additional steps are needed.
+  - If security labels are supported and you are unable to define them, they can be disabled for development purposes:
+    - `cp docker-compose.no-security-override.yml docker-compose.override.yml`
 - Clone this repository recursively (`git clone --recursive https://github.com/NASA-AMMOS/anms.git`)
 - Setup Volume containing PKI configuration (certificate chains and private keys):
   - `./create_volume.sh ./puppet/modules/apl_test/files/anms/tls`
@@ -98,7 +113,6 @@ Choose the appropriate docker, podman or podman-compose commands in the directio
   - `docker compose -f testenv-compose.yml build`
   - `podman compose -f testenv-compose.yml build`
   - `podman-compose --podman-build-args='--format docker' -f testenv-compose.yml build`
-  
 - Start System using one of the following:
   - `docker compose -f docker-compose.yml up -d`
   - `podman compose -f docker-compose.yml up -d`
@@ -126,11 +140,11 @@ To restart the system, use the 'up' and 'down' commands as described in the prev
 
 The top-level `docker-compose.yml` uses the environment defined by the sibling file `.env`.  Note: If using the legacy/deprecated build.sh script, that script may additionally override some environment variables.
 
-Two principal options of the compose configuration, which are both defaulted to empty text, are:
+The principal options of the compose configuration are:
 
 * `DOCKER_IMAGE_PREFIX` which controls any image name prefix added to all ANMS images.
   For a local build, this can be left empty, but for builds intended to be pushed to a Docker image registry this can be set to the full path on the registry before the image names (e.g. `DOCKER_IMAGE_PREFIX=some.host.example.com:5000/path/to/images`).
-
+* `HOST_SOCKDIR` which controls the source of the bind mount on `amp-manager` container for its transport socket. This can either be a volume name, for inter-container or non-root user use, or an absolute path on the host filesystem, used in the production deployment.
 
 
 ### AMP Database Querying