Merge branch 'main' into 228-add-ignore-scripts-argument-sonarqube

Author: BrianSipos

.env

@@ -1,3 +1,11 @@
+# This .env file is used by all compose commands for test and development usage.
+# NOTE: Production deployments via Puppet use an alternative version of this file generated by the tools.
+
+# Choose which profile(s) to run.
+#  If no profiles are set, a 'light' configuration will be started without the UI components
+#  Available profiles include 'full' (UI) and 'dev' (aeveloper tools such as adminer)
+COMPOSE_PROFILES=full,dev
+
 # Port Services; Uncomment below lines to override default mappings
 #AUTHNZ_PORT=8084
 #AUTHNZ_HTTPS_PORT=8443

.github/workflows/anms-core.yaml

@@ -30,3 +30,34 @@ jobs:
         run: |
           FAIL_SRC=0
           flake8 src || FAIL_SRC=$?
+  anms-core_integration-test:
+    runs-on: ubuntu-24.04
+    env:
+      ANMS_COMPOSE_OPTS: -f docker-compose.yml --profile light
+      TEST_COMPOSE_OPTS: -f anms-core/integration_test/docker-compose.yml
+      HOST_SOCKDIR: sockdir
+      CTR_SOCKDIR: /var/tmp/nm
+      DOCKER_CMD: docker
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Build ANMS
+        run: docker compose ${ANMS_COMPOSE_OPTS} build
+      - name: Build TEST
+        run: docker compose ${TEST_COMPOSE_OPTS} build
+      - name: Build Volume
+        run: |
+          ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls
+          sudo mkdir /run/anms
+      - name: run
+        run: |
+          docker compose  ${ANMS_COMPOSE_OPTS} up -d --force-recreate --wait --wait-timeout 600
+          docker compose ${TEST_COMPOSE_OPTS} run test-fixture
+      - name: after_script
+        run: |
+          if [ "${CI_JOB_STATUS}" = 'failed' ]; then
+            docker logs anms-core
+          fi
+

README.md

@@ -49,7 +49,16 @@ If your computer is behind a network proxy, this may cause issues related to usi
 Though ANMS can be run behind a proxy; building the ANMS Docker images from behind a network proxy may result in errors.
 
 The first steps in each of the container image `Dockerfile` is to attempt to download an APLNIS root CA to validate the APLNIS HTTPS proxy.
-When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.
+When building images outside of the APLNIS, this download will gracefully fail and the image will not be able to run within the APLNIS.  The URL for this certificate can be changed for users requiring equivalent functionality on their own networks.
+
+### Special Notes on Podman
+
+If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+
+Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+
+Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+
 
 ### Upgrading ANMS
 
@@ -63,29 +72,35 @@ The following command sequence uses standard Docker commands to stop all contain
 docker stop $(docker ps -q); docker rm $(docker ps --all -q); docker system prune -f; docker volume prune -f
 ```
 
-### Deployment Scenario
-
-The current ANMS capability is designed to run on `localhost` and on a development virtual machine.
-This guide presumes that you can either connect via a VMRC remote console or with ssh tunnelling to the machine, hence the use of `localhost` in db connection information and in URLs.
-If you deploy this to a VM, you will need to replace `localhost` with the hostname of the machine where it is deployed.
 
-### Special Notes on Podman
+## ANMS build and deploy
 
-If not otherwise specified, most commands in this document allow podman and docker to be used interchangeably. It is also possible to install an alias (provided in most package managers) to map `docker` to `podman` if desired.
+## Quickstart
 
-Podman, running as a standard user, is typically unable to bind to **low-numbered ports**. It is recommended to edit the `.env` file and uncomment the lines at top for AUTHNZ_PORT and AUTHNZ_HTTPS_PORT to remap those services to a higher port number.  In the directions below, you would then use for example http://localhost:8084 and https://localhost:8443 instead of the default.
+`./quickstart.sh`
 
-Note: If running on a system where **SELinux** is enabled, the system will not start if the appropriate security groups have not been defined. As an alternative, the `security_opt` sections can be commented out in the *-compose.yml files if required.
+The quickstart script will configure, build, and start the ANMS system for the first time.  See comments in the script for additional details, including optional ENV variables to override default behavior.
 
+To stop the system use `podman compose -f testenv-compose.yml -f docker-compose.yml down`.
 
-## ANMS build and deploy
+To start the system in the future use `podman compose -f testenv-compose.yml up` and `podman compose up`.
 
+## Manual Startup
 Choose the appropriate docker, podman or podman-compose commands in the directions below as appropriate for your system.
 
-- Select appropriate profile(s) as desired. 
-  - If no profiles are set, a "light" deployment of the ANMS focused on browser-less API-only ANMS users.
-  - For a full deployment: `export COMPOSE_PROFILES=full`
-  - For a full deployment with additional developer tools: `export COMPOSE_PROFILES=full,dev`
+- Edit `.env` file as appropriately
+  - Select appropriate profile(s) as desired. 
+    - Core ANMS services are always started.
+    - The 'full' profile starts up all UI and related services.
+    - The 'dev' profile adds development tools, such as adminer
+    - Profiles can be set with COMPOSE_PROFILES in the .env file. The default includes full and dev profiles.
+  - Adjust network ports as necessary to avoid any conflicts or permissions issues.
+    - For rootless podman, the AUTHNZ_* ports must be changed to higher number ports to avoid permissions issues. 
+    - The corresponding lines can be uncommented in .env.
+- SELinux Security Labels Setup
+  - If your system does not support security labels, no additional steps are needed.
+  - If security labels are supported and you are unable to define them, they can be disabled for development purposes:
+    - `cp docker-compose.no-security-override.yml docker-compose.override.yml`
 - Clone this repository recursively (`git clone --recursive https://github.com/NASA-AMMOS/anms.git`)
 - Setup Volume containing PKI configuration (certificate chains and private keys):
   - `./create_volume.sh ./puppet/modules/apl_test/files/anms/tls`
@@ -98,7 +113,6 @@ Choose the appropriate docker, podman or podman-compose commands in the directio
   - `docker compose -f testenv-compose.yml build`
   - `podman compose -f testenv-compose.yml build`
   - `podman-compose --podman-build-args='--format docker' -f testenv-compose.yml build`
-  
 - Start System using one of the following:
   - `docker compose -f docker-compose.yml up -d`
   - `podman compose -f docker-compose.yml up -d`

anms-core/anms/routes/adms/adm.py

@@ -56,7 +56,7 @@
 AdmData = adm_data.AdmData
 DataModel = data_model_view.DataModel
 
-ACCEPT_FILE_CONTENT_TYPE = "application/octet-stream"
+ACCEPT_FILE_CONTENT_TYPE = ["application/octet-stream","application/yang" ]
 
 
 class RequestError(BaseModel):
@@ -136,8 +136,10 @@ async def handle_adm(admset: ace.AdmSet, adm_file: ace.models.AdmModule, session
         if not replace:
             logger.info('Not replacing existing ADM name %s', adm_file.norm_name)
             return []
+        data_rec = None
+        async with get_async_session() as session:
+            data_rec,_ = await AdmData.get(data_model_view.data_model_id,session)
 
-        data_rec = await AdmData.get(data_model_view.data_model_id)
         if data_rec:
             # Compare old and new contents
             logger.info("Checking existing ADM name %s", adm_file.norm_name)
@@ -194,7 +196,7 @@ async def update_adm(file: UploadFile, request: Request):
     message = ""
     error_details = []  # This is used to store the comparison details between the old adm and the new adm
     # Check if not application/json
-    if file.content_type != ACCEPT_FILE_CONTENT_TYPE:
+    if file.content_type not in ACCEPT_FILE_CONTENT_TYPE:
         message = f"Expect {ACCEPT_FILE_CONTENT_TYPE}. Received: {file.content_type}"
         status_code = status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
         logger.error(message)
@@ -205,7 +207,7 @@ async def update_adm(file: UploadFile, request: Request):
         try:
             adm_file_contents = await file.read()
             try:
-                adm_file = admset.load_from_data(io.BytesIO(adm_file_contents).getvalue(), del_dupe=False)
+                adm_file = admset.load_from_data(io.StringIO(adm_file_contents.decode('utf-8')), del_dupe=False)
             except Exception as err:
                 adm_file = None
                 status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
@@ -214,19 +216,21 @@ async def update_adm(file: UploadFile, request: Request):
 
             if adm_file:
                 logger.info("Adm name: %s", adm_file.norm_name)
+                data_rec = None
                 # get data_model_id
-                data_model_rec, error_message = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name )
-                if error_message:
-                    raise Exception(error_message)
-
-
-                data_rec, error_message = await AdmData.get(data_model_rec.data_model_id )
-                if error_message:
-                    raise Exception(error_message)
+                async with get_async_session() as session:
+                    data_model_rec = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name, session )
+                    if data_model_rec == None:
+                        logger.info("new ADM dont compare" )
+                    else:
+                        data_rec,_ = await AdmData.get(data_model_rec.data_model_id,session )
+                        if data_rec == None:
+                            logger.warning("ADM not in DB can't compare")
+                    
                 # Compare with existing adm
                 if data_rec:
                     # Compare old and new contents
-                    old_adm = admset.load_from_data(io.BytesIO(data_rec.data).getvalue(), del_dupe=False)
+                    old_adm = admset.load_from_data(io.StringIO(data_rec.data.decode('utf-8')), del_dupe=False)
                     status_code = status.HTTP_200_OK
                     if not comp.compare_adms(old_adm, adm_file):
                         message = f"Updating existing adm {adm_file.norm_name}"
@@ -235,9 +239,10 @@ async def update_adm(file: UploadFile, request: Request):
                         # reload adm_set
                         admset.db_session().close()
                         admset = ace.AdmSet(cache_dir=False)
-                        adm_file = admset.load_from_data(io.BytesIO(adm_file_contents), del_dupe=False)
+                        adm_file = admset.load_from_data(io.StringIO(adm_file_contents.decode('utf-8')), del_dupe=False)
                     else: # if its the same nothing else to be done
-                        logger.info("Duplicate ADM add attempted")
+                        logger.warning("Duplicate ADM add attempted")
+                        message = "Duplicate ADM add attempted"
                         response = JSONResponse(status_code=status_code,
                                                 content={"message": message, "error_details": error_details})
                         return response
@@ -277,7 +282,7 @@ async def update_adm(file: UploadFile, request: Request):
             try:
                 async with get_async_session() as session:
                     # get data_model_id 
-                    
+                    data_model_rec = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name, session )
                     # Save the adm file of the new adm
                     data = {"enumeration": data_model_rec.data_model_id, "data": adm_file_contents}
                     response, error_message = await AdmData.add_data(data, session)

anms-core/integration_test/Dockerfile

@@ -21,56 +21,22 @@
 ##
 
 # Combine containers for anms-core and its test fixture
-version: '3.9'
+name: anms-integration-test
 
 networks:
-  default:
-    name: ${DOCKER_CTR_PREFIX}anms
-    driver_opts:
-      com.docker.network.bridge.name: br-${DOCKER_CTR_PREFIX}anms
-      com.docker.network.driver.mtu: 65535
+  anms:
+    external: true
+      
 
 services:
   # External dependencies first
-  postgres:
-    hostname: postgres
-    image: ${DOCKER_IMAGE_PREFIX}amp-sql:${DOCKER_IMAGE_TAG}
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
-      POSTGRES_DB: ${DB_NAME}
-  mqtt-broker:
-    hostname: mqtt-broker
-    image: ${DOCKER_IMAGE_PREFIX}mqtt-broker:${DOCKER_IMAGE_TAG}
-  transcoder:
-    hostname: transcoder
-    image: ${DOCKER_IMAGE_PREFIX}transcoder:${DOCKER_IMAGE_TAG}
-    depends_on:
-      mqtt-broker:
-        condition: service_healthy
-
-  # anms-core built from *this* working copy (not prebuilt image)
-  anms-core:
-    hostname: anms-core
-    build:
-      context: ..
-    volumes: 
-      - /var/run/docker.sock:/var/run/docker.sock
-    depends_on:
-      postgres:
-        condition: service_healthy
-      mqtt-broker:
-        condition: service_healthy
-    environment:
-      DB_HOST: postgres
-      DB_USER: ${DB_USER}
-      DB_PASSWORD: ${DB_PASSWORD}
-      DB_NAME: ${DB_NAME}
-
   test-fixture:
     hostname: test-fixture
     build:
-      context: .
-    depends_on:
-      anms-core:
-        condition: service_healthy
+      context: ../..
+      dockerfile: anms.Containerfile
+      target: anms-core-integration
+    networks:
+      - anms
+    depends_on:  {}
+    

anms-core/integration_test/docker-compose.yml

@@ -38,7 +38,7 @@
   const requestTimeOut = 3000; //milliseconds
   const axios = require('axios');
   const FormData = require('form-data');
-  const ACCEPTED_ADM_TYPE = 'application/json';
+  const ACCEPTED_ADM_TYPE = 'application/octet-stream';
 
   exports.getAll = async function (req, res, next) {
     try {
@@ -97,7 +97,7 @@
   exports.upload = async function (req, res, next) {
       const usersReqHeader = utils.createAuthenticationHeader(req);
       const file = req.file;
-
+      
       if (!_.isNull(file) && file.mimetype != ACCEPTED_ADM_TYPE) {
         return res.status(415).json({"message": `Not support this ${file.mimetype}`});
       }
@@ -126,6 +126,7 @@
       });
       if (_.isNil(response) || _.isNil(response.data) || _.isNil(response.data.message)) {
         response.status = 500;
+        console.error(response);
         response.data = {"message": "Internal Server Error"};
       }
       return res.status(response.status).json(response.data);

anms-ui/server/components/adms.js

@@ -260,6 +260,16 @@ EXPOSE 5555/tcp
 HEALTHCHECK --start-period=10s --interval=60s --timeout=10s --retries=20 \
     CMD ["curl", "-sq", "-o/dev/null", "http://localhost:5555/hello"]
 
+# for anms-core integration test
+FROM yarn-base AS anms-core-integration
+
+# Install node+yarn from upstream
+RUN npm install --ignore-scripts -g newman
+
+COPY anms-core/integration_test /root/
+WORKDIR /root
+CMD ["./run_test.sh"]
+
 
 # Build on more permissive CentOS image
 # Run on RHEL UBI image