Merge remote-tracking branch 'origin/main' into feature/177-ghcr_container_publishing

Author: DavidEdell

.github/workflows/anms-core.yaml

@@ -30,3 +30,34 @@ jobs:
         run: |
           FAIL_SRC=0
           flake8 src || FAIL_SRC=$?
+  anms-core_integration-test:
+    runs-on: ubuntu-24.04
+    env:
+      ANMS_COMPOSE_OPTS: -f docker-compose.yml --profile light
+      TEST_COMPOSE_OPTS: -f anms-core/integration_test/docker-compose.yml
+      HOST_SOCKDIR: sockdir
+      CTR_SOCKDIR: /var/tmp/nm
+      DOCKER_CMD: docker
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Build ANMS
+        run: docker compose ${ANMS_COMPOSE_OPTS} build
+      - name: Build TEST
+        run: docker compose ${TEST_COMPOSE_OPTS} build
+      - name: Build Volume
+        run: |
+          ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls
+          sudo mkdir /run/anms
+      - name: run
+        run: |
+          docker compose  ${ANMS_COMPOSE_OPTS} up -d --force-recreate --wait --wait-timeout 600
+          docker compose ${TEST_COMPOSE_OPTS} run test-fixture
+      - name: after_script
+        run: |
+          if [ "${CI_JOB_STATUS}" = 'failed' ]; then
+            docker logs anms-core
+          fi
+

anms-core/anms/routes/adms/adm.py

@@ -56,7 +56,7 @@
 AdmData = adm_data.AdmData
 DataModel = data_model_view.DataModel
 
-ACCEPT_FILE_CONTENT_TYPE = "application/octet-stream"
+ACCEPT_FILE_CONTENT_TYPE = ["application/octet-stream","application/yang" ]
 
 
 class RequestError(BaseModel):
@@ -136,8 +136,10 @@ async def handle_adm(admset: ace.AdmSet, adm_file: ace.models.AdmModule, session
         if not replace:
             logger.info('Not replacing existing ADM name %s', adm_file.norm_name)
             return []
+        data_rec = None
+        async with get_async_session() as session:
+            data_rec,_ = await AdmData.get(data_model_view.data_model_id,session)
 
-        data_rec = await AdmData.get(data_model_view.data_model_id)
         if data_rec:
             # Compare old and new contents
             logger.info("Checking existing ADM name %s", adm_file.norm_name)
@@ -194,7 +196,7 @@ async def update_adm(file: UploadFile, request: Request):
     message = ""
     error_details = []  # This is used to store the comparison details between the old adm and the new adm
     # Check if not application/json
-    if file.content_type != ACCEPT_FILE_CONTENT_TYPE:
+    if file.content_type not in ACCEPT_FILE_CONTENT_TYPE:
         message = f"Expect {ACCEPT_FILE_CONTENT_TYPE}. Received: {file.content_type}"
         status_code = status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
         logger.error(message)
@@ -205,7 +207,7 @@ async def update_adm(file: UploadFile, request: Request):
         try:
             adm_file_contents = await file.read()
             try:
-                adm_file = admset.load_from_data(io.BytesIO(adm_file_contents).getvalue(), del_dupe=False)
+                adm_file = admset.load_from_data(io.StringIO(adm_file_contents.decode('utf-8')), del_dupe=False)
             except Exception as err:
                 adm_file = None
                 status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
@@ -214,19 +216,21 @@ async def update_adm(file: UploadFile, request: Request):
 
             if adm_file:
                 logger.info("Adm name: %s", adm_file.norm_name)
+                data_rec = None
                 # get data_model_id
-                data_model_rec, error_message = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name )
-                if error_message:
-                    raise Exception(error_message)
-
-
-                data_rec, error_message = await AdmData.get(data_model_rec.data_model_id )
-                if error_message:
-                    raise Exception(error_message)
+                async with get_async_session() as session:
+                    data_model_rec = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name, session )
+                    if data_model_rec == None:
+                        logger.info("new ADM dont compare" )
+                    else:
+                        data_rec,_ = await AdmData.get(data_model_rec.data_model_id,session )
+                        if data_rec == None:
+                            logger.warning("ADM not in DB can't compare")
+                    
                 # Compare with existing adm
                 if data_rec:
                     # Compare old and new contents
-                    old_adm = admset.load_from_data(io.BytesIO(data_rec.data).getvalue(), del_dupe=False)
+                    old_adm = admset.load_from_data(io.StringIO(data_rec.data.decode('utf-8')), del_dupe=False)
                     status_code = status.HTTP_200_OK
                     if not comp.compare_adms(old_adm, adm_file):
                         message = f"Updating existing adm {adm_file.norm_name}"
@@ -235,9 +239,10 @@ async def update_adm(file: UploadFile, request: Request):
                         # reload adm_set
                         admset.db_session().close()
                         admset = ace.AdmSet(cache_dir=False)
-                        adm_file = admset.load_from_data(io.BytesIO(adm_file_contents), del_dupe=False)
+                        adm_file = admset.load_from_data(io.StringIO(adm_file_contents.decode('utf-8')), del_dupe=False)
                     else: # if its the same nothing else to be done
-                        logger.info("Duplicate ADM add attempted")
+                        logger.warning("Duplicate ADM add attempted")
+                        message = "Duplicate ADM add attempted"
                         response = JSONResponse(status_code=status_code,
                                                 content={"message": message, "error_details": error_details})
                         return response
@@ -277,7 +282,7 @@ async def update_adm(file: UploadFile, request: Request):
             try:
                 async with get_async_session() as session:
                     # get data_model_id 
-                    
+                    data_model_rec = await DataModel.get(adm_file.ns_model_enum, adm_file.ns_org_name, session )
                     # Save the adm file of the new adm
                     data = {"enumeration": data_model_rec.data_model_id, "data": adm_file_contents}
                     response, error_message = await AdmData.add_data(data, session)

anms-core/integration_test/Dockerfile

@@ -21,56 +21,22 @@
 ##
 
 # Combine containers for anms-core and its test fixture
-version: '3.9'
+name: anms-integration-test
 
 networks:
-  default:
-    name: ${DOCKER_CTR_PREFIX}anms
-    driver_opts:
-      com.docker.network.bridge.name: br-${DOCKER_CTR_PREFIX}anms
-      com.docker.network.driver.mtu: 65535
+  anms:
+    external: true
+      
 
 services:
   # External dependencies first
-  postgres:
-    hostname: postgres
-    image: ${DOCKER_IMAGE_PREFIX}amp-sql:${DOCKER_IMAGE_TAG}
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
-      POSTGRES_DB: ${DB_NAME}
-  mqtt-broker:
-    hostname: mqtt-broker
-    image: ${DOCKER_IMAGE_PREFIX}mqtt-broker:${DOCKER_IMAGE_TAG}
-  transcoder:
-    hostname: transcoder
-    image: ${DOCKER_IMAGE_PREFIX}transcoder:${DOCKER_IMAGE_TAG}
-    depends_on:
-      mqtt-broker:
-        condition: service_healthy
-
-  # anms-core built from *this* working copy (not prebuilt image)
-  anms-core:
-    hostname: anms-core
-    build:
-      context: ..
-    volumes: 
-      - /var/run/docker.sock:/var/run/docker.sock
-    depends_on:
-      postgres:
-        condition: service_healthy
-      mqtt-broker:
-        condition: service_healthy
-    environment:
-      DB_HOST: postgres
-      DB_USER: ${DB_USER}
-      DB_PASSWORD: ${DB_PASSWORD}
-      DB_NAME: ${DB_NAME}
-
   test-fixture:
     hostname: test-fixture
     build:
-      context: .
-    depends_on:
-      anms-core:
-        condition: service_healthy
+      context: ../..
+      dockerfile: anms.Containerfile
+      target: anms-core-integration
+    networks:
+      - anms
+    depends_on:  {}
+    

anms-core/integration_test/docker-compose.yml

@@ -38,7 +38,7 @@
   const requestTimeOut = 3000; //milliseconds
   const axios = require('axios');
   const FormData = require('form-data');
-  const ACCEPTED_ADM_TYPE = 'application/json';
+  const ACCEPTED_ADM_TYPE = 'application/octet-stream';
 
   exports.getAll = async function (req, res, next) {
     try {
@@ -97,7 +97,7 @@
   exports.upload = async function (req, res, next) {
       const usersReqHeader = utils.createAuthenticationHeader(req);
       const file = req.file;
-
+      
       if (!_.isNull(file) && file.mimetype != ACCEPTED_ADM_TYPE) {
         return res.status(415).json({"message": `Not support this ${file.mimetype}`});
       }
@@ -126,6 +126,7 @@
       });
       if (_.isNil(response) || _.isNil(response.data) || _.isNil(response.data.message)) {
         response.status = 500;
+        console.error(response);
         response.data = {"message": "Internal Server Error"};
       }
       return res.status(response.status).json(response.data);

anms-ui/server/components/adms.js

@@ -260,6 +260,16 @@ EXPOSE 5555/tcp
 HEALTHCHECK --start-period=10s --interval=60s --timeout=10s --retries=20 \
     CMD ["curl", "-sq", "-o/dev/null", "http://localhost:5555/hello"]
 
+# for anms-core integration test
+FROM yarn-base AS anms-core-integration
+
+# Install node+yarn from upstream
+RUN npm install --ignore-scripts -g newman
+
+COPY anms-core/integration_test /root/
+WORKDIR /root
+CMD ["./run_test.sh"]
+
 
 # Build on more permissive CentOS image
 # Run on RHEL UBI image

anms.Containerfile

@@ -1,4 +1,4 @@
-# ION Configuration File for Node N1, ipn:1
+# ION Configuration File for Node N1, ipn:1, hostname manager
 ## begin ionadmin
 1 1 ''
 s
@@ -12,12 +12,13 @@ m horizon +0
 ## begin bpadmin
 1
 e 1
+w 1
 a scheme ipn 'ipnfw' 'ipnadminep'
 a endpoint ipn:1.0 x
 a endpoint ipn:1.1 x
 a endpoint ipn:1.4 x
-a endpoint ipn:1.6 x
-a endpoint ipn:1.7 x
+a endpoint ipn:1.6 q
+a endpoint ipn:1.7 q
 
 
 # Protocols
@@ -28,19 +29,17 @@ a protocol tcp 1400 100
 a induct udp 0.0.0.0:4556 udpcli
 
 # Outducts
-a outduct udp 127.0.0.1 udpclo
-a outduct udp ION-AGENT2:4556 udpclo
-a outduct udp ION-AGENT3:4556 udpclo
-
+a outduct udp 10.5.0.101:4556 udpclo
+a outduct udp 10.5.0.102:4556 udpclo
+a outduct udp 10.5.0.103:4556 udpclo
 
 s
 ## end bpadmin
 
 ## begin ipnadmin
-a plan 1 udp/127.0.0.1
-a plan 2 udp/ION-AGENT2:4556
-a plan 3 udp/ION-AGENT3:4556
-
+a plan 1 udp/10.5.0.101:4556
+a plan 2 udp/10.5.0.102:4556
+a plan 3 udp/10.5.0.103:4556
 
 ## end ipnadmin